xdecoder/Instruct-X-Decoder

1import os2 3from transformers import CLIPTokenizer, CLIPTokenizerFast4from transformers import AutoTokenizer5 6from .registry import lang_encoders7from .registry import is_lang_encoder8 9 10def build_lang_encoder(config_encoder, tokenizer, verbose, **kwargs):11    model_name = config_encoder['NAME']12 13    if not is_lang_encoder(model_name):14        raise ValueError(f'Unkown model: {model_name}')15 16    return lang_encoders(model_name)(config_encoder, tokenizer, verbose, **kwargs)17 18 19def build_tokenizer(config_encoder):20    tokenizer = None21    os.environ['TOKENIZERS_PARALLELISM'] = 'true'22    if config_encoder['TOKENIZER'] == 'clip':23        pretrained_tokenizer = config_encoder.get(24            'PRETRAINED_TOKENIZER', 'openai/clip-vit-base-patch32'25        )26        tokenizer = CLIPTokenizer.from_pretrained(pretrained_tokenizer)27        tokenizer.add_special_tokens({'cls_token': tokenizer.eos_token})28    elif config_encoder['TOKENIZER'] == 'clip-fast':29        pretrained_tokenizer = config_encoder.get(30            'PRETRAINED_TOKENIZER', 'openai/clip-vit-base-patch32'31        )32        tokenizer = CLIPTokenizerFast.from_pretrained(pretrained_tokenizer, from_slow=True)33    else:34        tokenizer = AutoTokenizer.from_pretrained(config_encoder['TOKENIZER'])35 36    return tokenizer37