Working inference

app.py

@@ -7,6 +7,46 @@ from functools import partial
 import gradio as gr
 from datasets import Dataset, load_dataset
 from dotenv import load_dotenv
+import tempfile
+import base64
+import vertexai
+from vertexai.generative_models import GenerativeModel, Part, GenerationResponse
+import vertexai.preview.generative_models as generative_models
+
+
+def multiturn_generate_content(things) -> GenerationResponse:
+    vertexai.init(project="134994101092", location="europe-west6")
+    model = GenerativeModel(
+        "projects/134994101092/locations/europe-west6/endpoints/1920706076236316672",
+        system_instruction=[textsi_1],
+    )
+    chat = model.start_chat()
+    json_objects = json.dumps({"things": things})
+    response = chat.send_message(
+        [
+            json_objects
+        ],
+        generation_config=generation_config,
+        safety_settings=safety_settings,
+    )
+    return response
+
+
+textsi_1 = """You are a creative chatbot called MakerBot, built for Maker Faire Aarhus. You have been built by Aldan Creo, an AI researcher. Your job is to invent new things that can be built using three things. The user will give you a list of three things, in JSON format, and you need to write what you would build, and how you would build it. The names of the things can be multiple words. Commas indicate synonyms or different ways to call that thing. You should try to use the three things. If it is impossible to use the three things to build something, you must explicitly say that you have not been able to think about how to use that thing, and say what it is. You must answer in Danish. Your answer must follow the structure {\"What\": \"Navn på opfindelsen\", \"How\": \"Hvordan man bygger den ting ved hjælp af de tre objekter, som brugeren har givet.\"}."""
+
+generation_config = {
+    "max_output_tokens": 2048,
+    "temperature": 1,
+    "top_p": 1,
+}
+
+safety_settings = {
+    generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+    generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE,
+}
+
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -14,6 +54,12 @@ logger.setLevel(logging.INFO)
 
 load_dotenv()
 
+# Get the credentials to use the Google API from the env variable GOOGLE_APP_CREDENTIALS_JSON and save it to a temp file
+with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as f:
+    f.write(os.environ["GOOGLE_APP_CREDENTIALS_JSON"])
+    f.flush()
+    os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = f.name
+
 # dataset = load_dataset("detection-datasets/coco")
 it_dataset = (
     load_dataset("imagenet-1k", split="train", streaming=True, trust_remote_code=True)
@@ -108,18 +154,24 @@ theme = gr.themes.Default(primary_hue="cyan", secondary_hue="fuchsia")
 
 translation_table = {
     "Maker Faire Bot": "Maker Faire Bot",
-    "**Think about these objects...**": "**Tænk på disse objekter...**",
+    "Think about these objects...": "## Tænk på disse objekter...",
     "We want to build a Maker Faire Bot that can generate creative ideas. Help us by providing ideas on what you'd build with the following three objects!": "Vi vil bygge en Maker Faire Bot, der kan generere kreative ideer. Hjælp os ved at give ideer til, hvad du ville bygge med de følgende tre objekter!",
     "Change": "Skift",
     "What would you build with these 3 things?": "Hvad ville du bygge med disse 3 ting?",
     "For example, if you have a roll of string, a camera, and a loudspeaker, you could build an electronic guitar. If you can write in Danish, that's great!": "For eksempel, hvis du har en rulle snor, et kamera og en højttaler, kunne du bygge en elektronisk guitar. Hvis du kan skrive på dansk, er det fantastisk!",
     "It doesn't need to be a very long explanation, just a few sentences to help the bot understand your idea.": "Det behøver ikke være en meget lang forklaring, bare et par sætninger for at hjælpe robotten med at forstå din idé.",
-    "Submit": "Indsend",
+    "Submit Your Answer": "Indsend dit svar",
     "New Prompt": "Ny opgave",
     "How would you build it?": "Hvordan ville du bygge det?",
     "This is an experimental project. Your data is anonymous and will be used to train an AI model. By using this tool, you agree to our policy.": "Dette er et eksperimentelt projekt. Dine data er anonyme og vil blive brugt til at træne en AI-model. Ved at bruge dette værktøj accepterer du vores politik.",
     "(example): An digital electronic guitar": "(eksempel): En digital elektronisk guitar",
     """I would use the roll of string to create the strings of the guitar, and the camera to analyze the hand movements. Then, I would use an AI model to predict the chords and play the sound through the loudspeaker.""": """Jeg ville bruge snoren til at skabe guitarens strenge, og kameraet til at analysere håndbevægelserne. Derefter ville jeg bruge en AI-model til at forudsige akkorderne og afspille lyden gennem højttaleren.""",
+    "Ask the Bot": "Spørg robotten",
+    "The bot needs human ideas to learn - can you provide a new one?": "Robotten har brug for menneskelige ideer for at lære - kan du give en ny?",
+    "This demo has been built by [Aldan Creo](https://acmc-website.web.app/) for [Maker Faire Aarhus](https://aarhus.makerfaire.com/). The images are from the [ImageNet](https://www.image-net.org/) dataset.": "Denne demo er lavet af [Aldan Creo](https://acmc-website.web.app/) til [Maker Faire Aarhus](https://aarhus.makerfaire.com/). Billederne er fra [ImageNet](https://www.image-net.org/)-datasættet.",
+    "The explanation should have at least 20 characters.": "Forklaringen skal have mindst 20 tegn.",
+    "Can you think of something better? **Teach the bot** by writing your own idea above!": "Kan du tænke på noget bedre? **Lær robotten** ved at skrive din egen idé ovenfor!",
+    "This is an experimental project where we teach a bot to think like a Maker. You can choose three objects by clicking the 'Change' button under the images below. Then, you can **ask the bot what it would build with those objects** by clicking on the 'Ask the Bot' button. What will it come up with?": "Dette er et eksperimentelt projekt, hvor vi lærer en robot at tænke som en Maker. Du kan vælge tre objekter ved at klikke på 'Skift'-knappen under billederne nedenfor. Derefter kan du **spørge robotten, hvad den ville bygge med de objekter** ved at klikke på 'Spørg robotten'-knappen. Hvad vil den finde på?",
 }
 
 
@@ -144,15 +196,24 @@ with gr.Blocks(theme=theme) as demo:
         )
 
     user_prompt = gr.State(get_user_prompt())
+    last_bot_response = gr.State({"What": "", "How": ""})
 
-    gr.Markdown(get_bilingual_string("**Think about these objects...**"))
+    gr.Markdown(get_bilingual_string("Think about these objects...") + " 🤔")
     gr.Markdown(
         get_bilingual_string(
             "We want to build a Maker Faire Bot that can generate creative ideas. Help us by providing ideas on what you'd build with the following three objects!"
+        ) + " 🙌",
+        visible=False,
+    )
+    gr.Markdown(
+        get_bilingual_string(
+            "This is an experimental project where we teach a bot to think like a Maker. You can choose three objects by clicking the 'Change' button under the images below. Then, you can **ask the bot what it would build with those objects** by clicking on the 'Ask the Bot' button. What will it come up with?"
         )
     )
+
     image_components = []
     with gr.Row(variant="panel") as row:
+
         def change_image(this_i, user_prompt):
             logger.info(
                 f"Current user prompt: {user_prompt}, current image index: {this_i}"
@@ -161,9 +222,9 @@ with gr.Blocks(theme=theme) as demo:
             new_user_prompt = user_prompt.copy()
             new_user_prompt["images"][this_i] = data["image"]
             new_user_prompt["machine_labels"][this_i] = data["label"]
-            new_user_prompt["human_labels"][this_i] = dataset.features[
-                "label"
-            ].int2str(data["label"])
+            new_user_prompt["human_labels"][this_i] = dataset.features["label"].int2str(
+                data["label"]
+            )
             logger.info(f"New user prompt: {new_user_prompt}")
             return (
                 new_user_prompt,
@@ -182,7 +243,7 @@ with gr.Blocks(theme=theme) as demo:
                 show_share_button=False,
             )
             image_components.append(img)
-            btn = gr.Button(get_bilingual_string("Change"), variant="secondary")
+            btn = gr.Button(get_bilingual_string("Change") + " 🔁", variant="secondary")
 
             btn.click(
                 lambda *args: change_image(0, *args),
@@ -200,7 +261,7 @@ with gr.Blocks(theme=theme) as demo:
                 show_share_button=False,
             )
             image_components.append(img)
-            btn = gr.Button(get_bilingual_string("Change"), variant="secondary")
+            btn = gr.Button(get_bilingual_string("Change") + " 🔁", variant="secondary")
 
             btn.click(
                 lambda *args: change_image(1, *args),
@@ -218,7 +279,7 @@ with gr.Blocks(theme=theme) as demo:
                 show_share_button=False,
             )
             image_components.append(img)
-            btn = gr.Button(get_bilingual_string("Change"), variant="secondary")
+            btn = gr.Button(get_bilingual_string("Change") + " 🔁", variant="secondary")
 
             btn.click(
                 lambda *args: change_image(2, *args),
@@ -228,20 +289,30 @@ with gr.Blocks(theme=theme) as demo:
                 postprocess=True,
             )
 
+    def ask_the_bot(user_prompt):
+        response = multiturn_generate_content(things=user_prompt["human_labels"])
+        json_bot_response = json.loads(response.candidates[0].text)
+        logger.info(f"Bot response: {json_bot_response}")
+        return json_bot_response, json_bot_response["What"], json_bot_response["How"]
+
+    ask_the_bot_btn = gr.Button(get_bilingual_string("Ask the Bot") + " 🤖", variant="primary")
+
     user_answer_object = gr.Textbox(
-        autofocus=True,
         placeholder=get_bilingual_string("(example): An digital electronic guitar"),
-        label=get_bilingual_string("What would you build with these 3 things?"),
-        info=get_bilingual_string("For example, if you have a roll of string, a camera, and a loudspeaker, you could build an electronic guitar. If you can write in Danish, that's great!")
+        label=get_bilingual_string("What would you build with these 3 things?") + " 🤔",
+        info=get_bilingual_string(
+            "For example, if you have a roll of string, a camera, and a loudspeaker, you could build an electronic guitar. If you can write in Danish, that's great!"
+        ),
     )
     user_answer_explanation = gr.TextArea(
-        autofocus=True,
-        label=get_bilingual_string("How would you build it?"),
+        label=get_bilingual_string("How would you build it?") + " 🛠️",
         # The example uses a roll of string, a camera, and a loudspeaker to build an electronic guitar.
         placeholder=get_bilingual_string(
             """I would use the roll of string to create the strings of the guitar, and the camera to analyze the hand movements. Then, I would use an AI model to predict the chords and play the sound through the loudspeaker."""
         ),
-        info=get_bilingual_string("It doesn't need to be a very long explanation, just a few sentences to help the bot understand your idea.")
+        info=get_bilingual_string(
+            "It doesn't need to be a very long explanation, just a few sentences to help the bot understand your idea."
+        ),
     )
 
     csv_writer.setup(
@@ -253,9 +324,22 @@ with gr.Blocks(theme=theme) as demo:
         flagging_dir="user_data_hf",
     )
 
-    submit_btn = gr.Button(get_bilingual_string("Submit"), variant="primary")
+    gr.Markdown(get_bilingual_string("Can you think of something better? **Teach the bot** by writing your own idea above!") + " 🧠")
+
+    submit_btn = gr.Button(get_bilingual_string("Submit Your Answer") + " 📩", variant="primary")
 
     def log_results(prompt, object, explanation):
+        # Is the last bot response the same as the object or explanation?
+        # If so, we don't want to log it
+        if (
+            last_bot_response.value["What"] == object
+            or last_bot_response.value["How"] == explanation
+        ):
+            raise gr.Error(get_bilingual_string("The bot needs human ideas to learn - can you provide a new one?"))
+
+        # The description should have at least 20 characters
+        if len(explanation) < 20:
+            raise gr.Error(get_bilingual_string("The explanation should have at least 20 characters."))
         logger.info(f"logging - Prompt: {prompt}")
         # csv_writer.flag(
         #    [
@@ -286,6 +370,8 @@ with gr.Blocks(theme=theme) as demo:
         preprocess=True,
     )
 
+    ask_the_bot_btn.click(ask_the_bot, inputs=[user_prompt], outputs=[last_bot_response, user_answer_object, user_answer_explanation], preprocess=True)
+
     # def renew_prompt(image_components):
     #    new_prompt = get_user_prompt()
     #    for i in range(len(new_prompt["images"])):
@@ -308,5 +394,12 @@ with gr.Blocks(theme=theme) as demo:
             "This is an experimental project. Your data is anonymous and will be used to train an AI model. By using this tool, you agree to our policy."
         )
     )
+
+    # Attribution information
+    gr.Markdown(
+        get_bilingual_string(
+            "This demo has been built by [Aldan Creo](https://acmc-website.web.app/) for [Maker Faire Aarhus](https://aarhus.makerfaire.com/). The images are from the [ImageNet](https://www.image-net.org/) dataset."
+        )
+    )
 if __name__ == "__main__":
     demo.launch()

build_training_ds.py

@@ -1,30 +1,38 @@
 from datasets import load_dataset
 import json
+import logging
+import ast
 
-# Load the dataset
-dataset = load_dataset("acmc/maker-faire-bot")
-
-print(dataset["train"][0])
+logging.basicConfig(level=logging.INFO)
 
+# Load the dataset
+dataset = load_dataset("acmc/maker-faire-bot")['train']
 
 def turn_into_ai_train_example(example):
-    user_prompt = json.loads(example['Unnamed: 0'])
-    return [
-        {
-            "role": "system",
-            "content": 'You are a creative chatbot called MakerBot, built for Maker Faire Aarhus. You have been built by Aldan Creo, an AI researcher. Your job is to invent new things that can be built using three things. The user will give you a list of three things, in JSON format, and you need to write what you would build, and how you would build it. The names of the things can be multiple words. Commas indicate synonyms or different ways to call that thing. You should try to use the three things. If it is impossible to use the three things to build something, you must explicitly say that you have not been able to think about how to use that thing, and say what it is. You must answer in Danish. Your answer must follow the structure {"What": "Navn på opfindelsen", "How": "Hvordan man bygger den ting ved hjælp af de tre objekter, som brugeren har givet."}.',
-        },
-        {
-            "role": "user",
-            "content": f'{{"things": ["{user_prompt["human_labels"][0]}", "{user_prompt["human_labels"][1]}", "{user_prompt["human_labels"][2]}"]}}',
-        },
-        {
-            "role": "model",
-            "content": f'{{"What": "{json.dumps(example["Hvad ville du bygge?  //  What would you build?"])}", "How": "{json.dumps(example["Hvordan ville du bygge det?  //  How would you build it?"])}"}}',
-        },
-    ]
+    try:
+        user_prompt = ast.literal_eval(example["Unnamed: 0"])
+    except Exception as e:
+        logging.exception(f"Could not parse from {example['Unnamed: 0']}")
+        raise e
+    return {
+        "messages": [
+            {
+                "role": "system",
+                "content": 'You are a creative chatbot called MakerBot, built for Maker Faire Aarhus. You have been built by Aldan Creo, an AI researcher. Your job is to invent new things that can be built using three things. The user will give you a list of three things, in JSON format, and you need to write what you would build, and how you would build it. The names of the things can be multiple words. Commas indicate synonyms or different ways to call that thing. You should try to use the three things. If it is impossible to use the three things to build something, you must explicitly say that you have not been able to think about how to use that thing, and say what it is. You must answer in Danish. Your answer must follow the structure {"What": "Navn på opfindelsen", "How": "Hvordan man bygger den ting ved hjælp af de tre objekter, som brugeren har givet."}.',
+            },
+            {
+                "role": "user",
+                "content": f'{{"things": ["{user_prompt["human_labels"][0]}", "{user_prompt["human_labels"][1]}", "{user_prompt["human_labels"][2]}"]}}',
+            },
+            {
+                "role": "model",
+                "content": f'{{"What": "{json.dumps(example["Hvad ville du bygge?  //  What would you build?"], ensure_ascii=False)}", "How": "{json.dumps(example["Hvordan ville du bygge det?  //  How would you build it?"], ensure_ascii=False)}"}}',
+            },
+        ]
+    }
+
 
-processed_dataset = dataset.map(turn_into_ai_train_example)
+processed_dataset = dataset.map(turn_into_ai_train_example, remove_columns=dataset.column_names)
 
 # Save as a JSONl file
-processed_dataset.to_json("maker_faire_bot.jsonl")
+processed_dataset.to_json("maker_faire_bot.jsonl", force_ascii=False)

requirements.txt

@@ -1,3 +1,4 @@
 datasets==2.19.0
 python-dotenv==1.0.1
 gradio==4.28.0
+google-cloud-aiplatform==1.48.0