bigscience/petals-api

1import argparse2import os3 4import psutil5import torch.backends.quantized6import torch.nn as nn7import transformers8from hivemind.utils.logging import get_logger, use_hivemind_log_handler9from huggingface_hub import Repository10from tqdm.auto import tqdm11 12from src import BloomModel13from src.client import DistributedBloomConfig14from src.bloom.from_pretrained import CLIENT_BRANCH, BLOCK_BRANCH_PREFIX15use_hivemind_log_handler("in_root_logger")16logger = get_logger(__file__)17 18DTYPE_MAP = dict(bfloat16=torch.bfloat16, float16=torch.float16, float32=torch.float32, auto="auto")19 20 21if __name__ == "__main__":22    parser = argparse.ArgumentParser(description="Load bloom layers and convert to 8-bit using torch quantization.")23 24    parser.add_argument("--model", type=str, default="bigscience/bloom-6b3", help="Model name for from_pretrained")25    parser.add_argument("--revision", type=str, default=None, help="Optional commit id from HF hub")26    parser.add_argument("--torch_dtype", type=str, default="auto", help="Load initial model in this dtype")27    parser.add_argument("--output_path", type=str, default="./converted_model", help="Track output repo to this folder")28    parser.add_argument("--output_repo", type=str, default="bigscience/test-bloomd", help="Push to this HF hub repo")29    parser.add_argument("--client_branch", type=str, default=CLIENT_BRANCH, help="Save client version to this branch")30    parser.add_argument(31        "--block_branch_prefix", type=str, default=BLOCK_BRANCH_PREFIX, help="Save blocks to branches with this prefix"32    )33    parser.add_argument(34        "--commit_message", type=str, default="push-o-matic", help="Use this commit message for all parts"35    )36    parser.add_argument("--use_auth_token", type=str, default=None, help="auth token for from_pretrained")37    args = parser.parse_args()38 39    free_ram_gb = psutil.virtual_memory().available / 2**3040    if args.model == "bigscience/bloom" and free_ram_gb < 400:41        logger.warning(f"ACHTUNG! converting bloom-176b will use up 350-400GB RAM, you have {free_ram_gb:.3f} free")42 43    assert args.torch_dtype in DTYPE_MAP, f"torch_dtype must be one of {list(DTYPE_MAP.keys())}"44    if os.path.exists(args.output_path) and (45        len(os.listdir(args.output_path)) != 0 or not os.path.isdir(args.output_path)46    ):47        raise FileExistsError(f"Output path {args.output_path} already exists and is not an empty directory")48 49    logger.info(f"Loading source model {args.model} (this may take a few minutes)")50    config = DistributedBloomConfig.from_pretrained(51        args.model, use_auth_token=args.use_auth_token, revision=args.revision52    )53    config.dht_prefix = args.output_repo54 55    model = BloomModel.from_pretrained(56        args.model, use_auth_token=args.use_auth_token, revision=args.revision, torch_dtype=DTYPE_MAP[args.torch_dtype]57    )58    tokenizer = transformers.AutoTokenizer.from_pretrained(59        args.model, use_auth_token=args.use_auth_token, revision=args.revision60    )61    os.makedirs(args.output_path, exist_ok=True)62 63    repo = Repository(args.output_path, clone_from=args.output_repo, use_auth_token=args.use_auth_token)64    repo.git_pull()65 66    transformer_blocks = model.h67    logger.info(68        f"Saving transformer blocks to {args.output_repo}@{args.block_branch_prefix}0"69        f" - {args.output_repo}@{args.block_branch_prefix}{len(transformer_blocks)}"70    )71    for i, block in enumerate(tqdm(transformer_blocks)):72        repo.git_checkout(args.client_branch, create_branch_ok=True)73        with repo.commit(74            commit_message=args.commit_message, branch=args.block_branch_prefix + str(i), track_large_files=True75        ):76            torch.save(block.state_dict(), "./pytorch_model.bin")77 78    logger.info(f"Saving client-side modules to {args.output_repo}@{args.client_branch}")79    repo.git_checkout(args.client_branch, create_branch_ok=True)80    with repo.commit(commit_message=args.commit_message, branch=args.client_branch, track_large_files=True):81        model.h = nn.ModuleList()82        model.save_pretrained(".")83        tokenizer.save_pretrained(".")84        config.save_pretrained(".")85 86    logger.info(f"Converted {args.model} and pushed to {args.output_repo}")87