CintraAI/code-chunker

1import tiktoken2import json3 4def count_tokens(string: str, encoding_name: str) -> int:5    """Returns the number of tokens in a text string."""6    encoding = tiktoken.encoding_for_model(encoding_name)7    num_tokens = len(encoding.encode(string))8    return num_tokens9 10def load_json(json_file):11    with open(json_file) as f:12        return json.load(f)13