node-loki/quantized-retrieval

1 2from datasets import load_dataset3import numpy as np4from usearch.index import Index5from sentence_transformers.util import quantize_embeddings6 7dataset = load_dataset("mixedbread-ai/wikipedia-2023-11-embed-en-pre-1", split="train")8embeddings = np.array(dataset["emb"], dtype=np.float32)9 10int8_embeddings = quantize_embeddings(embeddings, "int8")11index = Index(ndim=1024, metric="ip", dtype="i8")12index.add(np.arange(len(int8_embeddings)), int8_embeddings)13index.save("wikipedia_int8_usearch_1m.index")14