cwenzi/neuroflow-cpp

1import json
2
3path = r'D:\neuroflow-C++\configs\tokenizer_128k.json'
4d = json.load(open(path, 'r', encoding='utf-8'))
5vocab = d['vocab']
6
7expected = set(range(128000))
8actual = set(vocab.values())
9missing = sorted(list(expected - actual))
10
11print('Missing IDs:', len(missing))
12print('Range:', missing[0], '-', missing[-1])
13
14for mid in missing:
15    token = '<unused_' + str(mid) + '>'
16    if token in vocab:
17        print('Collision at', token, '-> using alt')
18        token = '<fill_' + str(mid) + '>'
19    vocab[token] = mid
20
21d['vocab'] = vocab
22d['vocab_size'] = 128000
23
24with open(path, 'w', encoding='utf-8') as f:
25    json.dump(d, f, ensure_ascii=False, indent=2)
26
27# Verify
28all_ids = set(vocab.values())
29still_missing = set(range(128000)) - all_ids
30print('After fix - tokens:', len(vocab), 'missing IDs:', len(still_missing))
31