ElliotJenkins/MLLM_path

1#!/usr/bin/env python2# Copyright (c) OpenMMLab. All rights reserved.3from setuptools import find_packages, setup4 5 6def readme():7    with open('README.md', encoding='utf-8') as f:8        content = f.read()9    return content10 11 12version_file = 'xtuner/version.py'13 14 15def get_version():16    with open(version_file) as f:17        exec(compile(f.read(), version_file, 'exec'))18    return locals()['__version__']19 20 21def parse_requirements(fname='requirements.txt', with_version=True):22    """Parse the package dependencies listed in a requirements file but strips23    specific versioning information.24 25    Args:26        fname (str): path to requirements file27        with_version (bool, default=False): if True include version specs28 29    Returns:30        List[str]: list of requirements items31 32    CommandLine:33        python -c "import setup; print(setup.parse_requirements())"34    """35    import re36    import sys37    from os.path import exists38    require_fpath = fname39 40    def parse_line(line):41        """Parse information from a line in a requirements text file."""42        if line.startswith('-r '):43            # Allow specifying requirements in other files44            target = line.split(' ')[1]45            for info in parse_require_file(target):46                yield info47        else:48            info = {'line': line}49            if line.startswith('-e '):50                info['package'] = line.split('#egg=')[1]51            elif '@git+' in line:52                info['package'] = line53            else:54                # Remove versioning from the package55                pat = '(' + '|'.join(['>=', '==', '>']) + ')'56                parts = re.split(pat, line, maxsplit=1)57                parts = [p.strip() for p in parts]58 59                info['package'] = parts[0]60                if len(parts) > 1:61                    op, rest = parts[1:]62                    if ';' in rest:63                        # Handle platform specific dependencies64                        # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies65                        version, platform_deps = map(str.strip,66                                                     rest.split(';'))67                        info['platform_deps'] = platform_deps68                    else:69                        version = rest70                    info['version'] = (op, version)71            yield info72 73    def parse_require_file(fpath):74        with open(fpath) as f:75            for line in f.readlines():76                line = line.strip()77                if line and not line.startswith('#'):78                    yield from parse_line(line)79 80    def gen_packages_items():81        if exists(require_fpath):82            for info in parse_require_file(require_fpath):83                parts = [info['package']]84                if with_version and 'version' in info:85                    parts.extend(info['version'])86                if not sys.version.startswith('3.4'):87                    # apparently package_deps are broken in 3.488                    platform_deps = info.get('platform_deps')89                    if platform_deps is not None:90                        parts.append(';' + platform_deps)91                item = ''.join(parts)92                yield item93 94    packages = list(gen_packages_items())95    return packages96 97 98if __name__ == '__main__':99    setup(100        name='xtuner',101        version=get_version(),102        description=('An efficient, flexible and full-featured toolkit for '103                     'fine-tuning large models'),104        long_description=readme(),105        long_description_content_type='text/markdown',106        author='XTuner Contributors',107        author_email='openmmlab@gmail.com',108        keywords='large language model, parameter-efficient fine-tuning',109        url='https://github.com/InternLM/xtuner',110        packages=find_packages(),111        include_package_data=True,112        classifiers=[113            'Development Status :: 4 - Beta',114            'License :: OSI Approved :: Apache Software License',115            'Operating System :: OS Independent',116            'Programming Language :: Python :: 3',117            'Programming Language :: Python :: 3.8',118            'Programming Language :: Python :: 3.9',119            'Programming Language :: Python :: 3.10',120            'Topic :: Utilities',121        ],122        # Python maximum version <3.11, to support mpi4py-mpich123        python_requires='>=3.8, <3.11',124        license='Apache License 2.0',125        install_requires=parse_requirements('requirements/runtime.txt'),126        extras_require={127            'all':128            parse_requirements('requirements.txt'),129            'deepspeed':130            parse_requirements('requirements/runtime.txt') +131            parse_requirements('requirements/deepspeed.txt'),132            'modelscope':133            parse_requirements('requirements/runtime.txt') +134            parse_requirements('requirements/modelscope.txt'),135        },136        zip_safe=False,137        entry_points={'console_scripts': ['xtuner = xtuner:cli']})138