<jupyter_start><jupyter_text>IVPQ<jupyter_code>class IVPQIndex():
    def __init__(self, vectors, labels):
        self.dimention = vectors.shape[1]
        self.vectors = vectors.astype('float32')
        self.labels = labels


    def build(self, number_of_partition=8, search_in_x_partitions=2, subvector_size=8):
        quantizer = faiss.IndexFlatL2(self.dimention)
        self.index = faiss.IndexIVFPQ(quantizer, 
                                      self.dimention, 
                                      number_of_partition, 
                                      search_in_x_partitions, 
                                      subvector_size)
        self.index.train(self.vectors)
        self.index.add(self.vectors)
        
    def query(self, vectors, k=10):
        distances, indices = self.index.search(vectors, k) 
        return [self.labels[i] for i in indices[0]]

index = IVPQIndex(data["vector"], data["name"])
index.build()
movie_index = 90
movie_vector = data['vector'][movie_index:movie_index+1]
print(f"The most simillar movies to {data['name'][movie_index]} are:")
index.query(movie_vector)<jupyter_output>The most simillar movies to Nightmare Before Christmas, The (1993) are:
{'fraction_non_alphanumeric': 0.10696517412935323, 'fraction_numerical': 0.014096185737976783, 'mean_word_length': 2.4485714285714284, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '623302', 'license': 'MIT', 'n_tokens_mistral': 391, 'n_tokens_neox': 358, 'n_words': 67, 'path': 'Lectures/search_in_practice-approximate_nearest_neighbors/Product Quantization.ipynb', 'repo_name': 'eyaltrabelsi/Monty-Hall-Problem', 'size': '1206'}