<jupyter_start><jupyter_text>Mini test<jupyter_code>test = [
"Juice Del Valle Frut Pet Peach 1 L Juice Del Valle Frut Pet Peach 1 L",
"Elseve Total 5 Extra Professional Serum 15 mL Elseve Total 5 Extra Professional Serum 15 mL",
"Corbalan Canned Mixed Skewer 500 g Corbalan Canned Mixed Skewer 500 g",
"Pedigree Dog Food Puppies Medium and Large Breeds 3 kg With Vitamins and Minerals that help you stay strong and healthy. With Natural Fibers for optimal digestion. With Calcium for growth. With protein for strong muscles. Textured grains that help reduce the formation of tartar and keep your teeth and gums healthy.",
"Kendall 7/8 BC Sock Without Tip M 1322 Kendall 7/8 BC Sock Without Tip M 1322",
"Johnsons Roma Soap 80 g Johnsons Roma Soap 80 g",
"Sensodyne Extra Fresh Rinse 250 ml Sensodyne Extra Fresh Rinse 250 ml",
"Kelloggs Granola And Honey Biscuit 120 Gr Kelloggs Granola And Honey Biscuit 120 Gr",
"Multivitaminico Vita Force AZ 120 Cáps - Voxx Multivitaminico Vita Force AZ 120 Cáps - Voxx",
"Bioext Queravit Megadose 15 mL Bioext Queravit Megadose 15 mL",
"Moo Yogurt Type Skyr Coco Without Lactose Moo Yogurt Type Skyr Coco Without Lactose",
"Beer 600 ml Choose your beer.",
"Bienn 4mg 10 Tablets Bienn 4Mg 10 Tablets",
"Dental Brush B Indicator Nº40 2 U Dental Brush Indicator Oral-B, White Teeth And Healthy Gums. It has bristles with rounded tips and with Indicator Plus system, which indicates the moment of brush change, its handle is rubberized with comfort grip, which provides more safety and comfort during brushing. Enjoy the Promotion Take 2, Pay 1. Product with colors and / or assorted prints. Shipping According to Stock Availability. (Key Words: Toothpaste, Toothpaste, Oral Health, Oral Hygiene, Anticaries, Toothbrush, Tooth Hiding)",
"Nestlé Flan Strawberry Dessert Set 200 g Nestlé Flan Strawberry Dessert Set 200 g (Key Words: Dairy / Morning)",
"Wickbold Scooby Doo Integral Tube 300 g The Wickbold Scooby Doo wholegrain biscuit was made with wholemeal flour especially for those who follow a slimming diet and cannot do without the daily bisnaguinha. Healthy, soft and tasty, ideal for any time of the day.",
"Orthodontic Success Interdental Kit x 4 Units - PLU: 33455",
"Dark Chocolate With Tiramisu Kopenhagen Filling Dark Chocolate With Tiramisu Kopenhagen Filling",
"Naturafrig Bovine Palette Bovine Naturafrig Bulk Palette",
"Palmolive Naturals Soap Secret Seductive 90 g Palmolive Naturals Soap Secret Seductive 90 g. (Key Words: Body Care, Beauty Care, Beauty, Hygiene, Deodorant, Anti Perspirant, Antiperspirant, Soap, Soap, Bar Soap)",
]

categories = [
    "food",
    "health and beauty", 
    "cleaning supplies", 
    "canned foods", 
    "pet supplies", 
    "fashion apparel"
]
for t in test:
    print(t)
    for c in categories:
        print("\t", embedding_distance(t, c), c)
        
    print()<jupyter_output><empty_output><jupyter_text>Rappi data test<jupyter_code>import csv

labels = []
samples = []

# took out subcategory or else evaluation takes forever

with open("rappi_sub_categories.csv", newline='') as csvfile:
    reader = csv.DictReader(csvfile, delimiter=',')
    for row in reader:
        #labels.append(row['category'] + ' ' + row['subcategory'])
        labels.append(row['category'])
labels = list(set(labels))

with open("rappi_human.csv", newline='') as csvfile:
    reader = csv.DictReader(csvfile, delimiter=',')
    for row in reader:
        text = row['translated_title'] + ' ' + row['translated_description']
        label = row['category'] #+ ' ' + row['subcategory']
        if label not in labels:
            continue
            
        samples.append((text, label))
        
    
print(len(samples))
print(len(labels))
for sample in samples: 
    print(sample)
    predictions = [(embedding_distance(sample[0], l), l) for l in tqdm(labels)]
    predictions = sorted(predictions)
    print(predictions[:5])
    break<jupyter_output>0%|          | 0/182 [00:00<?, ?it/s]
{'fraction_non_alphanumeric': 0.08119011598587998, 'fraction_numerical': 0.024457892082702975, 'mean_word_length': 4.151948051948052, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '600853', 'license': 'Apache-2.0', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1208, 'n_words': 508, 'path': 'notebooks/Rappi Predict with model (for Sarah).ipynb', 'repo_name': 'glisten-ai/transformers', 'size': '3966'}