unpredictable/unpredictable_support-google-com

1# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""This loads the UnpredicTable-support-google-com dataset."""15 16import json17import os18import pandas as pd19 20import datasets21 22 23_DESCRIPTION = """\24The UnpredicTable dataset consists of web tables formatted as few-shot tasks for fine-tuning language models to improve their few-shot performance. For more details please see the accompanying dataset card.25"""26 27_LICENSE = "Apache 2.0"28 29_URL = "https://huggingface.co/datasets/unpredictable/unpredictable_support-google-com/resolve/main/unpredictable_support-google-com.jsonl"30 31logger = datasets.logging.get_logger(__name__)32 33 34class UnpredicTable(datasets.GeneratorBasedBuilder):35    """36    The UnpredicTable dataset consists of web tables formatted as few-shot tasks for fine-tuning language models to improve their few-shot performance. For more details please see the accompanying dataset card.37    """38 39    VERSION = datasets.Version("1.0.0")40 41    def _info(self):42        features = datasets.Features(43            {44                "task": datasets.Value("string"),45                "input": datasets.Value("string"),46                "output": datasets.Value("string"),47                "options": datasets.Sequence([datasets.Value("string")]),48                "pageTitle": datasets.Value("string"),49                "outputColName": datasets.Value("string"),50                "url": datasets.Value("string"),51                "wdcFile": datasets.Value("string")52            }53        )54        return datasets.DatasetInfo(55            description=_DESCRIPTION,56            features=features,57            license=_LICENSE,58        )59 60    def _split_generators(self, dl_manager):61        """Returns SplitGenerators."""62        data_dir = dl_manager.download_and_extract(_URL)63        return [64            datasets.SplitGenerator(65                name=datasets.Split.TRAIN,66                gen_kwargs={"filepath": data_dir},67            ),68        ]69 70    def _generate_examples(self, filepath):71        """Yields examples."""72        with open(filepath, encoding="utf-8") as f:73            for i, row in enumerate(f):74                data = json.loads(row)75                key = f"{data['task']}_{i}"76                yield key, {77                    "task": data["task"],78                    "input": data["input"],79                    "output": data["output"],80                    "options": data["options"],81                    "pageTitle": data["pageTitle"],82                    "outputColName": data["outputColName"],83                    "url": data["url"],84                    "wdcFile": data["wdcFile"],85                }