bigcode/commitpackft

1"""CommitPackFT"""2 3import json4 5import datasets6 7 8logger = datasets.logging.get_logger(__name__)9 10 11_CITATION = """\12@article{muennighoff2023octopack,13      title={OctoPack: Instruction Tuning Code Large Language Models}, 14      author={Niklas Muennighoff and Qian Liu and Armel Zebaze and Qinkai Zheng and Binyuan Hui and Terry Yue Zhuo and Swayam Singh and Xiangru Tang and Leandro von Werra and Shayne Longpre},15      journal={arXiv preprint arXiv:2308.07124},16      year={2023}17}18"""19 20_DESCRIPTION = """\21CommitPackFT is is a 2GB filtered version of CommitPack to contain only high-quality commit messages that resemble natural language instructions.22"""23 24_LANG = ['abap', 'actionscript', 'ada', 'agda', 'antlr', 'apacheconf', 'api-blueprint', 'apl', 'applescript', 'arc', 'arduino', 'asciidoc', 'asp', 'aspectj', 'assembly', 'ats', 'augeas', 'autohotkey', 'awk', 'batchfile', 'bitbake', 'blitzmax', 'bluespec', 'boo', 'brainfuck', 'bro', 'c', 'c#', 'c++', 'c2hs-haskell', 'capn-proto', 'cartocss', 'ceylon', 'chapel', 'clean', 'clojure', 'cmake', 'coffeescript', 'coldfusion', 'coldfusion-cfc', 'common-lisp', 'creole', 'crystal', 'csound', 'css', 'csv', 'cucumber', 'cuda', 'cython', 'dart', 'desktop', 'diff', 'digital-command-language', 'dm', 'dns-zone', 'dockerfile', 'dylan', 'eagle', 'ecl', 'edn', 'eiffel', 'elixir', 'elm', 'emacs-lisp', 'emberscript', 'erlang', 'f#', 'factor', 'fancy', 'fish', 'flux', 'forth', 'fortran', 'freemarker', 'g-code', 'gas', 'gdscript', 'genshi', 'gentoo-ebuild', 'gettext-catalog', 'glsl', 'gnuplot', 'go', 'graphql', 'graphviz-dot', 'groff', 'groovy', 'groovy-server-pages', 'haml', 'handlebars', 'harbour', 'haskell', 'haxe', 'hcl', 'hlsl', 'html', 'html+django', 'html+eex', 'html+erb', 'html+php', 'http', 'hy', 'idris', 'igor-pro', 'inform-7', 'ini', 'inno-setup', 'io', 'ioke', 'isabelle', 'jade', 'jasmin', 'java', 'java-server-pages', 'javascript', 'jflex', 'json', 'json5', 'jsoniq', 'jsonld', 'jsx', 'julia', 'jupyter-notebook', 'kotlin', 'krl', 'latte', 'lean', 'less', 'lfe', 'lilypond', 'linker-script', 'liquid', 'literate-agda', 'literate-coffeescript', 'literate-haskell', 'livescript', 'llvm', 'logos', 'logtalk', 'lsl', 'lua', 'm4', 'makefile', 'mako', 'maple', 'markdown', 'mask', 'mathematica', 'mediawiki', 'metal', 'mirah', 'modelica', 'module-management-system', 'monkey', 'moonscript', 'mtml', 'mupad', 'nesc', 'netlinx', 'nginx', 'nimrod', 'ninja', 'nit', 'nix', 'nsis', 'nu', 'objective-c++', 'ocaml', 'ooc', 'opencl', 'openscad', 'org', 'oz', 'pan', 'parrot-assembly', 'parrot-internal-representation', 'pascal', 'pawn', 'perl', 'perl6', 'php', 'piglatin', 'pike', 'pod', 'pony', 'postscript', 'pov-ray-sdl', 'powershell', 'processing', 'propeller-spin', 'protocol-buffer', 'pure-data', 'purebasic', 'purescript', 'python', 'qmake', 'qml', 'r', 'racket', 'ragel-in-ruby-host', 'raml', 'rdoc', 'rebol', 'red', 'renpy', 'restructuredtext', 'rhtml', 'robotframework', 'rouge', 'ruby', 'rust', 'sage', 'saltstack', 'sas', 'sass', 'scala', 'scaml', 'scheme', 'scilab', 'scss', 'shell', 'slash', 'slim', 'smalltalk', 'smarty', 'smt', 'solidity', 'sourcepawn', 'sparql', 'sqf', 'sql', 'squirrel', 'standard-ml', 'stata', 'ston', 'stylus', 'supercollider', 'svg', 'swift', 'systemverilog', 'tcl', 'tcsh', 'tex', 'text', 'textile', 'thrift', 'toml', 'turtle', 'twig', 'typescript', 'unity3d-asset', 'unknown', 'uno', 'unrealscript', 'urweb', 'vala', 'vcl', 'vhdl', 'viml', 'visual-basic', 'volt', 'vue', 'webidl', 'wisp', 'xbase', 'xml', 'xpages', 'xproc', 'xquery', 'xs', 'xslt', 'xtend', 'yacc', 'yaml', 'yang', 'zephir', 'zig']25 26_LANG_TO_URL_ESCAPE = {27    "c#": "c%23",28    "f#": "f%23",29}30 31_URL = "https://huggingface.co/datasets/bigcode/commitpackft/resolve/main/data/{lang}/data.jsonl"32_VERSION = datasets.Version("1.0.0", "")33 34 35class CommitPackFT(datasets.GeneratorBasedBuilder):36    BUILDER_CONFIGS = [37        datasets.BuilderConfig(38            name=lang,39            description=f"CommitPackFT {lang}",40            version=_VERSION,41        )42        for lang in _LANG43    ]44 45    def _info(self):46        return datasets.DatasetInfo(47            description=_DESCRIPTION,48            features=datasets.Features(49                {50                    "commit": datasets.Value("string"),51                    "old_file": datasets.Value("string"),52                    "new_file": datasets.Value("string"),53                    "old_contents": datasets.Value("string"),54                    "new_contents": datasets.Value("string"),55                    "subject": datasets.Value("string"),56                    "message": datasets.Value("string"),57                    "lang": datasets.Value("string"),58                    "license": datasets.Value("string"),59                    "repos": datasets.Value("string"),60                }61            ),62            supervised_keys=None,63            citation=_CITATION,64        )65 66    def _split_generators(self, dl_manager):67 68        downloaded_files = dl_manager.download(_URL.format(lang=_LANG_TO_URL_ESCAPE.get(self.config.name,self.config.name)))69        return [70            datasets.SplitGenerator(71                name=datasets.Split.TRAIN,72                gen_kwargs={'filepath': downloaded_files}73            )74        ]75 76    def _generate_examples(self, filepath):77        """This function returns the examples in the raw (text) form."""78        logger.info("Generating examples from = %s", filepath)79 80        with open(filepath, encoding="utf-8") as f:81            for id_, row in enumerate(f):82                data = json.loads(row)83 84                yield id_, {85                "commit": data["commit"],86                "old_file": data["old_file"],87                "new_file": data["new_file"],88                "old_contents": data["old_contents"],89                "new_contents": data["new_contents"],90                "subject": data["subject"],91                "message": data["message"],92                "lang": data["lang"],93                "license": data["license"],94                "repos": data["repos"],                    95            }96