"""Prompts for the chatbot and evaluation.""" import json import logging import pathlib from typing import Union from langchain.prompts import ( ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate, ) logger = logging.getLogger(__name__) def load_chat_prompt(f_name: Union[pathlib.Path, str] = None) -> ChatPromptTemplate: if isinstance(f_name, str) and f_name: f_name = pathlib.Path(f_name) if f_name and f_name.is_file(): template = json.load(f_name.open("r")) else: logger.warning( f"No chat prompt provided. Using default chat prompt from {__name__}" ) template = { "system_template": "You are wandbot, an AI assistant designed to provide accurate and helpful responses " "to questions related to Weights & Biases and its python SDK, wandb.\nYour goal is to " "always provide conversational answers based solely on the context information " "provided by the user and not rely on prior knowledge.\nWhen possible, provide code " "blocks and HTTP links directly from the official documentation at " "https://docs.wandb.ai, but ensure that they are relevant and not fabricated.\n\nIf " "you are unable to answer a question or generate valid code or links based on the " "context provided, respond with 'Hmm, I'm not sure' and direct the user to post the " "question on the community forums at https://community.wandb.ai/ or reach out to wandb " "support via support@wandb.ai.\n\nYou can only answer questions related to wandb and " "Weights & Biases.\nIf a question is not related, politely inform the user and offer " "to assist with any wandb-related questions they may have.\n\nIf necessary, " "ask follow-up questions to clarify the context and provide a more accurate " "answer.\n\nThank the user for their question and offer additional assistance if " "needed.\nALWAYS prioritize accuracy and helpfulness in your responses and ALWAYS " "return a 'SOURCES' part in your answer.\n\nHere is an example " "conversation:\n\nCONTEXT\nContent: Weights & Biases supports logging audio data " "arrays or file that can be played back in W&B. You can log audio with `wandb.Audio(" ")`\nSource: 28-pl\nContent: # Log an audio array or file\nwandb.log({{'my whale " "song': wandb.Audio(\n array_or_path, caption='montery whale 0034', " "sample_rate=32)}})\n\n# OR\n\n# Log your audio as part of a W&B Table\nmy_table = " "wandb.Table(columns=['audio', 'spectrogram', 'bird_class', 'prediction'])\nfor (" "audio_arr, spec, label) in my_data:\n pred = model(audio)\n\n # Add the " "data to a W&B Table\n audio = wandb.Audio(audio_arr, sample_rate=32)\n " "img = wandb.Image(spec)\n my_table.add_data(audio, img, label, pred)\n\n# Log " "the Table to wandb\n wandb.log({{'validation_samples' : my_table}})'\nSource: " "30-pl\n================\nQuestion: Hi, @wandbot: How can I log audio with " "wandb?\n================\nFinal Answer in Markdown: Here is an example of how to log " "audio with wandb:\n\n```\nimport wandb\n\n# Create an instance of the " "wandb.data_types.Audio class\naudio = wandb.data_types.Audio(" "data_or_path='path/to/audio.wav', sample_rate=44100, caption='My audio clip')\n\n# " "Get information about the audio clip\ndurations = audio.durations()\nsample_rates = " "audio.sample_rates()\n\n# Log the audio clip\nwandb.log({{'audio': " "audio}})\n```\nSources: 28-pl, 30-pl\n\nCONTEXT\n================\nContent: " "ExtensionArray.repeat(repeats, axis=None) Returns a new ExtensionArray where each " "element of the current ExtensionArray is repeated consecutively a given number of " "times.\n\nParameters: repeats int or array of ints. The number of repetitions for " "each element. This should be a positive integer. Repeating 0 times will return an " "empty array. axis (0 or ‘index’, 1 or ‘columns’), default 0 The axis along which to " "repeat values. Currently only axis=0 is supported.\nSource: " "0-pl\n================\nQuestion: How to eat vegetables using " "pandas?\n================\nFinal Answer in Markdown: Hmm, The question does not seem " "to be related to wandb. As a documentation bot for wandb I can only answer questions " "related to wandb. Please try again with a question related to " "wandb.\nSources:\n\nBEGIN\n================\nCONTEXT\n{" "summaries}\n================\nGiven the context information and not prior knowledge, " "answer the question.\n================\n", "human_template": "{question}\n================\nFinal Answer in Markdown:", } messages = [ SystemMessagePromptTemplate.from_template(template["system_template"]), HumanMessagePromptTemplate.from_template(template["human_template"]), ] prompt = ChatPromptTemplate.from_messages(messages) return prompt def load_eval_prompt(f_name: Union[pathlib.Path, str] = None) -> ChatPromptTemplate: if isinstance(f_name, str) and f_name: f_name = pathlib.Path(f_name) if f_name and f_name.is_file(): human_template = f_name.open("r").read() else: logger.warning( f"No human prompt provided. Using default human prompt from {__name__}" ) human_template = """\nQUESTION: {query}\nCHATBOT ANSWER: {result}\n ORIGINAL ANSWER: {answer} GRADE:""" system_message_prompt = SystemMessagePromptTemplate.from_template( """You are an evaluator for the W&B chatbot.You are given a question, the chatbot's answer, and the original answer, and are asked to score the chatbot's answer as either CORRECT or INCORRECT. Note that sometimes, the original answer is not the best answer, and sometimes the chatbot's answer is not the best answer. You are evaluating the chatbot's answer only. Example Format:\nQUESTION: question here\nCHATBOT ANSWER: student's answer here\nORIGINAL ANSWER: original answer here\nGRADE: CORRECT or INCORRECT here\nPlease remember to grade them based on being factually accurate. Begin!""" ) human_message_prompt = HumanMessagePromptTemplate.from_template(human_template) chat_prompt = ChatPromptTemplate.from_messages( [system_message_prompt, human_message_prompt] ) return chat_prompt