updating items and adding url classification option

app.py

@@ -4,11 +4,12 @@ import torch
 import requests
 import numpy as np
 from PIL import Image
+from io import BytesIO
 
 # Sidebar content
 sidebar_markdown = """
 
-We have several resources related to our new fashion models.
+Note, this demo can classify 300 items. If you didn't find what you're looking for, reach out to us on our [Community](https://join.slack.com/t/marqo-community/shared_invite/zt-2iab0260n-QJrZLUSOJYUifVxf964Gdw) and request an item to be added.
 
 ## Documentation
 📚 [Blog Post](https://www.marqo.ai/blog/search-model-for-fashion)
@@ -37,36 +38,57 @@ year = {2024}
 
 # List of fashion items
 items = [
-    "Leggings", "Jogger", "Palazzo", "Cargo", "Dresspants", "Chinos", 
-    "Dress", "Blouse", "T-Shirt", "Jeans", "Skirt", "Shorts", 
-    "Sweater", "Cardigan", "Tank Top", "Hoodie", "Coat", 
-    "Jacket", "Polo Shirt", "Crop Top", "Romper", 
-    "Overalls", "Blazer", "Sweatpants", "Vest", 
-    "Dungarees", "Poncho", "Bodysuit", "Maxi Dress", 
-    "Hat", "Sunglasses", "Glasses", "Shoes", "Sandals", 
-    "Heels", "Trainers", "Belt", "Tie", "Dress Shirt", 
-    "Boots", "Slippers", "Sneakers", "Insoles", "Socks", 
-    "Insulated Jacket", "Fleece", "Rain Jacket", "Running Jacket", 
-    "Windbreaker", "Shirt", "Graphic Top", "Sweatshirt", 
-    "Pant", "Legging", "Short", "Skort", "Brief", 
-    "Sports Bra", "Base Layer Top", "Base Layer Bottom", 
-    "Swimsuit", "Rashguard", "Water Shorts", "Cover Up", 
-    "Goggle", "Sun Hat", "Glove", "Mitten", "Leg Gaiter", 
-    "Bandana", "Neck Gaiter", "Balaclava", "Sunglass", 
-    "Watch", "Bag", "Boxer", "Swim Trunk", "Ring", 
-    "Necklace", "Earing", "Pocket Watch", "Smock", 
-    "Trouser", "Tuxedo", "Cufflinks", "Suspenders", 
-    "Handkerchief", "Scarf", "Wristband", "Beanie", 
-    "Fedora", "Beret", "Clutch Bag", "Crossbody Bag", 
-    "Duffel Bag", "Backpack", "Umbrella", "Flip Flops", 
-    "Espadrilles", "Loafers", "Moccasins", "Chelsea Boots", 
-    "Ankle Boots", "Bow Tie", "Tie Clip", "Hair Clip", 
-    "Headband", "Visor", "Baseball Cap", "Bucket Hat", 
-    "Fingerless Gloves", "Touchscreen Gloves", "Trench Coat", 
-    "Peacoat", "Parka", "Bomber Jacket", "Utility Vest", 
-    "Puffer Jacket", "Cape", "Shrug", "Kimono", "Sarong", 
-    "Apron", "Bikini", "Poncho", "Wristwatch", "Choker", 
-    "Brooch", "Anklet", "Toe Ring", "Waist Belt"
+    'abaya', 'anorak', 'apron', 'ascot tie',
+    'balaclava', 'ball gown', 'bandanna', 'baseball cap', 'bathing suit', 
+    'beanie', 'bedclothes', 'bell-bottoms', 'belt', 'beret', 
+    'Bermuda shorts', 'baby clothes', 'bib', 'bikini', 'blazer', 'bloomers', 'blouse', 'boa', 
+    'bonnet', 'boot', 'bow', 'bow tie', 'boxer shorts', 'boxers', 'bra', 
+    'bracelet', 'brassiere', 'breeches', 'briefs', 'buckle', 'button', 
+    'caftan', 'camisole', 'camouflage', 'cap', 
+    'cap and gown', 'cape', 'capris', 'cardigan', 'chemise', 'cloak', 'clogs', 
+    'coat', 'collar', 'corset', 'costume', 'coveralls', 
+    'cowboy boots', 'cowboy hat', 'cravat', 'crown', 'cuff', 'cuff links', 
+    'culottes', 'dashiki', 'diaper', 'dinner jacket', 'dirndl', 
+    'drawers', 'dress', 'dress shirt', 'duds', 'dungarees', 'earmuffs', 
+    'earrings', 'elastic', 'evening gown', 'fashion', 'fedora', 
+    'fez', 'flak jacket', 'flannel nightgown', 'flannel shirt', 'flip-flops', 
+    'formal wear', 'frock', 'fur', 'fur coat', 'gabardine', 'gaiters', 
+    'galoshes', 'garb', 'garters', 'getup', 'gilet', 
+    'girdle', 'glasses', 'gloves', 'gown', 'halter top', 'handbag', 
+    'handkerchief', 'hat', 'Hawaiian shirt', 'hazmat suit', 'headscarf', 
+    'helmet', 'hem', 'high heels', 'hoodie', 'hook and eye', 'hose', 
+    'hosiery', 'hospital gown', 'houndstooth', 'housecoat', 'jacket', 
+    'jeans', 'jersey', 'jewelry', 'jodhpurs', 'jumper', 'jumpsuit', 'kerchief', 
+    'khakis', 'kilt', 'kimono', 'kit', 'knickers', 'lab coat', 'lapel', 
+    'leather jacket', 'leg warmers', 'leggings', 'leotard', 'life jacket', 
+    'lingerie', 'loafers', 'loincloth', 'long johns', 'long underwear', 
+    'miniskirt', 'mittens', 'moccasins', 'muffler', 'muumuu', 'neckerchief', 
+    'necklace', 'nightgown', 'nightshirt', 'onesies', 'outerwear', 'outfit', 
+    'overalls', 'overcoat', 'overshirt', 'pajamas', 'pants', 
+    'pantsuit', 'pantyhose', 'parka', 'pea coat', 'peplum', 'petticoat', 
+    'pinafore', 'pleat', 'pocket', 'pocketbook', 'polo shirt', 'poncho', 
+    'poodle skirt', 'pullover', 'pumps', 'purse', 'raincoat', 
+    'ring', 'robe', 'rugby shirt', 'sandals', 'sari', 'sarong', 'scarf', 
+    'school uniform', 'scrubs', 'shawl', 'shirt', 
+    'shoes', 'shorts', 'shoulder pads', 'shrug', 'singlet', 'skirt', 
+    'slacks', 'slip', 'slippers', 'smock', 'snaps', 'sneakers', 'socks', 
+    'sombrero', 'spacesuit', 'stockings', 'stole', 'suit', 
+    'sun hat', 'sunbonnet', 'sundress', 'sunglasses', 'suspenders', 
+    'sweater', 'sweatpants', 'sweatshirt', 'sweatsuit', 'swimsuit', 
+    'T-shirt', 'tam', 'tank top', 'teddy', 'threads', 'tiara', 'tie', 
+    'tie clip', 'tights', 'toga', 'tog', 'top', 'top coat', 'top hat', 'train', 
+    'trench coat', 'trousers', 'trunks', 'tube top', 'tunic', 'turban', 
+    'turtleneck', 'turtleneck shirt', 'tutu', 'tuxedo', 'tweed jacket', 
+    'twin set', 'umbrella', 'underclothes', 'undershirt', 
+    'underwear', 'uniform', 'veil', 'Velcro', 'vest', 'vestments', 'visor', 
+    'waders', 'waistcoat', 'wear', 'wedding gown', 'Wellingtons', 'wetsuit', 
+    'white tie', 'wig', 'windbreaker', 'woolens', 'wrap', 'yoke', 'zipper', 
+    'zoris', 'jogger', 'palazzo', 'cargo', 'dresspants', 'chinos', 
+    'crop top', 'romper', 'insulated jacket', 'fleece', 'rain jacket', 
+    'running jacket', 'graphic top', 'pant', 'legging', 'skort', 'brief', 
+    'sports bra', 'water shorts', 'cover up', 'goggle', 'glove', 'mitten', 
+    'leg gaiter', 'neck gaiter', 'watch', 'bag', 'swim trunk', 
+    'pocket watch', 'insoles', "climbing shoes",
 ]
 
 
@@ -90,27 +112,28 @@ def generate_description(item):
         return f"An item of {item} worn on the hands"
     else:
         return f"A fashion item called {item}"
-
-
 items_desc = [generate_description(item) for item in items]
 text = tokenizer(items_desc)
 
-# Encode text features
-with torch.no_grad(), torch.cuda.amp.autocast():
+# Encode text features (unchanged)
+with torch.no_grad(), torch.amp.autocast('cuda'):
     text_features = model.encode_text(text)
     text_features /= text_features.norm(dim=-1, keepdim=True)
 
 # Prediction function
-def predict(inp):
-    image = preprocess_val(inp).unsqueeze(0)
+def predict(image, url):
+    if url:
+        response = requests.get(url)
+        image = Image.open(BytesIO(response.content))
+    
+    processed_image = preprocess_val(image).unsqueeze(0)
 
-    with torch.no_grad(), torch.cuda.amp.autocast():
-        image_features = model.encode_image(image)
+    with torch.no_grad(), torch.amp.autocast('cuda'):
+        image_features = model.encode_image(processed_image)
         image_features /= image_features.norm(dim=-1, keepdim=True)
 
         text_probs = (100 * image_features @ text_features.T).softmax(dim=-1)
         
-        # Sort the confidences and get the top 10
         sorted_confidences = sorted(
             {items[i]: float(text_probs[0, i]) for i in range(len(items))}.items(), 
             key=lambda x: x[1], 
@@ -118,13 +141,16 @@ def predict(inp):
         )
         top_10_confidences = dict(sorted_confidences[:10])
         
-    return top_10_confidences
+    return image, top_10_confidences
+
+# Clear function
+def clear_fields():
+    return None, ""
 
 # Gradio interface
 title = "Fashion Item Classifier with Marqo-FashionSigLIP"
-description = "Upload an image of a fashion item and classify it using [Marqo-FashionSigLIP](https://huggingface.co/Marqo/marqo-fashionSigLIP)!"
+description = "Upload an image or provide a URL of a fashion item to classify it using [Marqo-FashionSigLIP](https://huggingface.co/Marqo/marqo-fashionSigLIP)!"
 
-# Example image paths with thumbnails
 examples = [
     ["images/dress.jpg", "Dress"],
     ["images/sweatpants.jpg", "Sweatpants"],
@@ -152,14 +178,15 @@ with gr.Blocks(css="""
             gr.Markdown(" ", elem_id="vertical-line")  # Add an empty Markdown with a custom ID
         with gr.Column(scale=2):
             input_image = gr.Image(type="pil", label="Upload Fashion Item Image", height=312)
-            predict_button = gr.Button("Classify")
+            input_url = gr.Textbox(label="Or provide an image URL")
+            with gr.Row():
+                predict_button = gr.Button("Classify")
+                clear_button = gr.Button("Clear")
             gr.Markdown("Or click on one of the images below to classify it:")
             gr.Examples(examples=examples, inputs=input_image)
-        # with gr.Column(scale=3):            
             output_label = gr.Label(num_top_classes=6)
-            predict_button.click(predict, inputs=input_image, outputs=output_label)
+            predict_button.click(predict, inputs=[input_image, input_url], outputs=[input_image, output_label])
+            clear_button.click(clear_fields, outputs=[input_image, input_url])
         
-    
 # Launch the interface
-demo.launch(share=True)
-
+demo.launch()