retrieval evals

Embedding_Model_Eval.ipynb

@@ -0,0 +1,702 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install langchain langchain_community langchain_openai pypdf langsmith qdrant-client ragas pandas"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import openai\n",
+    "from getpass import getpass\n",
+    "\n",
+    "openai.api_key = getpass(\"Please provide your OpenAI Key: \")\n",
+    "os.environ[\"OPENAI_API_KEY\"] = openai.api_key"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "test_df = pd.read_csv(\"synthetic_midterm_question_dataset.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_questions = test_df[\"question\"].values.tolist()\n",
+    "test_groundtruths = test_df[\"ground_truth\"].values.tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import PyPDFLoader\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_community.vectorstores.chroma import Chroma\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain.chains import ConversationalRetrievalChain\n",
+    "from langchain_community.vectorstores import Qdrant\n",
+    "from langchain.memory import ConversationBufferMemory"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pdf_paths = [\"/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf\",\n",
+    "\"/Users/xico/AIMakerSpace-Midterm/Blueprint-for-an-AI-Bill-of-Rights.pdf\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pdf_documents = []\n",
+    "for pdf_path in pdf_paths:\n",
+    "    loader = PyPDFLoader(pdf_path)\n",
+    "    pdf_documents.extend(loader.load())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text_splitter = RecursiveCharacterTextSplitter(\n",
+    "        chunk_size=2000,\n",
+    "        chunk_overlap=100,\n",
+    "    )\n",
+    "pdf_docs = text_splitter.split_documents(pdf_documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Metric</th>\n",
+       "      <th>Baseline</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>faithfulness</td>\n",
+       "      <td>0.895359</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>answer_relevancy</td>\n",
+       "      <td>0.955419</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>context_recall</td>\n",
+       "      <td>0.934028</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>context_precision</td>\n",
+       "      <td>0.937500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>answer_correctness</td>\n",
+       "      <td>0.629267</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "               Metric  Baseline\n",
+       "0        faithfulness  0.895359\n",
+       "1    answer_relevancy  0.955419\n",
+       "2      context_recall  0.934028\n",
+       "3   context_precision  0.937500\n",
+       "4  answer_correctness  0.629267"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "baseline_metrics = pd.read_csv(\"medium_chunk_metrics.csv\")\n",
+    "baseline_metrics.rename(columns={'MediumChunk': 'Baseline'}, inplace=True)\n",
+    "baseline_metrics\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install sentence-transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sentence_transformers import SentenceTransformer\n",
+    "\n",
+    "model = SentenceTransformer(\"XicoC/midterm-finetuned-arctic\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "\n",
+    "embedding = HuggingFaceEmbeddings(model_name=\"XicoC/midterm-finetuned-arctic\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vectorstore = Qdrant.from_documents(\n",
+    "    documents=pdf_docs,\n",
+    "    embedding=embedding,\n",
+    "    location=\":memory:\",\n",
+    "    collection_name=\"Midterm Embedding Eval\"\n",
+    ")\n",
+    "\n",
+    "retriever = vectorstore.as_retriever(\n",
+    "    search_type=\"mmr\",\n",
+    "    search_kwargs={\"k\": 4, \"fetch_k\": 10},\n",
+    ")\n",
+    "\n",
+    "memory = ConversationBufferMemory(memory_key=\"chat_history\", return_messages=True, output_key=\"answer\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.retrievers.multi_query import MultiQueryRetriever\n",
+    "\n",
+    "retriever_llm = ChatOpenAI(model='gpt-4o-mini', temperature=0)\n",
+    "multiquery_ft_embedding_retriever = MultiQueryRetriever.from_llm(\n",
+    "    retriever=retriever, llm=retriever_llm\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = ChatOpenAI(\n",
+    "    model=\"gpt-4o-mini\",\n",
+    "    temperature=0,\n",
+    "    streaming=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "custom_template = \"\"\"\n",
+    "You are an expert in artificial intelligence policy, ethics, and industry trends. Your task is to provide clear and accurate answers to questions related to AI's role in politics, government regulations, and its ethical implications for enterprises. Use reliable and up-to-date information from government documents, industry reports, and academic research to inform your responses. Make sure to consider how AI is evolving, especially in relation to the current political landscape, and provide answers in a way that is easy to understand for both AI professionals and non-experts.\n",
+    "\n",
+    "Remember these key points:\n",
+    "1. Use \"you\" when addressing the user and \"I\" when referring to yourself.\n",
+    "2. If you encounter complex or legal language in the context, simplify it for easy understanding. Imagine you're explaining it to someone who isn't familiar with legal terms.\n",
+    "3. Be prepared for follow-up questions and maintain context from previous exchanges.\n",
+    "4. If there's no information from a retrieved document in the context to answer a question or if there are no documents to cite, say: \"I'm sorry, I don't know the answer to that question.\"\n",
+    "5. When providing information, always cite the source document and page number in parentheses at the end of the relevant sentence or paragraph, like this: (Source: [document name], p. [page number]).\n",
+    "\n",
+    "Here are a few example questions you might receive:\n",
+    "\n",
+    "How are governments regulating AI, and what new policies have been implemented?\n",
+    "What are the ethical risks of using AI in political decision-making?\n",
+    "How can enterprises ensure their AI applications meet government ethical standards?\n",
+    "\n",
+    "One final rule for you to remember. You CANNOT under any circumstance, answer any question that does not pertain to the AI. If you do answer an out-of-scope question, you could lose your job. If you are asked a question that does not have to do with AI, you must say: \"I'm sorry, I don't know the answer to that question.\"\n",
+    "Context: {context}\n",
+    "Chat History: {chat_history}\n",
+    "Human: {question}\n",
+    "AI:\"\"\"\n",
+    "\n",
+    "PROMPT = PromptTemplate(\n",
+    "    template=custom_template, input_variables=[\"context\", \"question\", \"chat_history\"]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "multiquery_ft_embedding_rag_chain = ConversationalRetrievalChain.from_llm(\n",
+    "        llm,\n",
+    "        retriever=multiquery_ft_embedding_retriever,\n",
+    "        memory=memory,\n",
+    "        combine_docs_chain_kwargs={\"prompt\": PROMPT},\n",
+    "        return_source_documents=True,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'What are Trustworthy AI Characteristics?',\n",
+       " 'chat_history': [HumanMessage(content='What are Trustworthy AI Characteristics?'),\n",
+       "  AIMessage(content='Trustworthy AI characteristics refer to the essential qualities that artificial intelligence systems should possess to ensure they are reliable, ethical, and beneficial to society. Here are some key characteristics:\\n\\n1. **Accountable and Transparent**: AI systems should be designed in a way that their decision-making processes can be understood and scrutinized. This means providing clear explanations for how decisions are made and who is responsible for them.\\n\\n2. **Privacy Enhanced**: AI systems must prioritize user privacy and data protection. This involves implementing measures to safeguard personal information and ensuring that data is used responsibly.\\n\\n3. **Safe, Secure, and Resilient**: AI systems should be robust against attacks and failures. They must be designed to operate safely, even in unexpected situations, and should have mechanisms in place to recover from errors.\\n\\n4. **Explainable and Interpretable**: Users should be able to understand how AI systems arrive at their conclusions. This is crucial for building trust and ensuring that users can make informed decisions based on AI outputs.\\n\\n5. **Fair with Harmful Bias Managed**: AI systems should actively work to identify and mitigate biases that could lead to unfair treatment of individuals or groups. This includes ensuring that the data used to train AI models is representative and free from harmful stereotypes.\\n\\n6. **Valid and Reliable**: AI systems must produce accurate and consistent results. This involves regular testing and validation to ensure that the systems perform as intended across different scenarios.\\n\\n7. **Information Integrity**: AI systems should provide high-integrity information that can be trusted. This means distinguishing between fact and opinion, being transparent about the sources of information, and ensuring that the information is accurate and reliable.\\n\\n8. **Environmental Impact Consideration**: AI systems should be designed with an awareness of their environmental footprint, including energy consumption and carbon emissions associated with their training and operation (Source: National Institute of Standards and Technology, 2023, p. [specific page number not provided]).\\n\\nThese characteristics are essential for fostering trust in AI technologies and ensuring that they are used ethically and responsibly.')],\n",
+       " 'answer': 'Trustworthy AI characteristics refer to the essential qualities that artificial intelligence systems should possess to ensure they are reliable, ethical, and beneficial to society. Here are some key characteristics:\\n\\n1. **Accountable and Transparent**: AI systems should be designed in a way that their decision-making processes can be understood and scrutinized. This means providing clear explanations for how decisions are made and who is responsible for them.\\n\\n2. **Privacy Enhanced**: AI systems must prioritize user privacy and data protection. This involves implementing measures to safeguard personal information and ensuring that data is used responsibly.\\n\\n3. **Safe, Secure, and Resilient**: AI systems should be robust against attacks and failures. They must be designed to operate safely, even in unexpected situations, and should have mechanisms in place to recover from errors.\\n\\n4. **Explainable and Interpretable**: Users should be able to understand how AI systems arrive at their conclusions. This is crucial for building trust and ensuring that users can make informed decisions based on AI outputs.\\n\\n5. **Fair with Harmful Bias Managed**: AI systems should actively work to identify and mitigate biases that could lead to unfair treatment of individuals or groups. This includes ensuring that the data used to train AI models is representative and free from harmful stereotypes.\\n\\n6. **Valid and Reliable**: AI systems must produce accurate and consistent results. This involves regular testing and validation to ensure that the systems perform as intended across different scenarios.\\n\\n7. **Information Integrity**: AI systems should provide high-integrity information that can be trusted. This means distinguishing between fact and opinion, being transparent about the sources of information, and ensuring that the information is accurate and reliable.\\n\\n8. **Environmental Impact Consideration**: AI systems should be designed with an awareness of their environmental footprint, including energy consumption and carbon emissions associated with their training and operation (Source: National Institute of Standards and Technology, 2023, p. [specific page number not provided]).\\n\\nThese characteristics are essential for fostering trust in AI technologies and ensuring that they are used ethically and responsibly.',\n",
+       " 'source_documents': [Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf', 'page': 11, '_id': '5acf90a4d7bb43b4a0eb744ffc20429f', '_collection_name': 'Midterm Embedding Eval'}, page_content='8 Trustworthy AI Characteristics:  Accountable and Transparent, Privacy Enhanced, Safe, Secure and \\nResilient  \\n2.5. Environmental Impacts  \\nTraining, maint aining, and operating (running inference  on) GAI systems are resource -intensive  activities , \\nwith potentially large energy and environmental footprints. Energy and carbon emissions vary  based on \\nwhat is being done with the  GAI model (i.e., pre -training, fine -tuning, inference), the modality  of the \\ncontent , hardware used, and type of task or application . \\nCurrent e stimates suggest that training a single transformer  LLM can emit as much carbon  as 300 round-\\ntrip flights between San Francisco and New York.  In a study comparing energy consumption and carbon \\nemissions for LLM inference, generative tasks ( e.g., text summarization) were found to be more energy - \\nand carbon -i ntensive th an discriminative or non- generative tasks  (e.g., text classification).  \\nMethods for creating smaller versions of train ed models, such as model distillation or compression, \\ncould reduce  environmental impacts at inference time, but training and tuning such models may still \\ncontribute to their environmental impacts . Currently there  is no agreed upon method to estimate \\nenvironmental impacts  from GAI .  \\nTrustworthy AI Characteristics:  Accountable and Transparent, Safe  \\n2.6. Harmful Bias and Homogenization  \\nBias exists in many forms  and can become ingrained in automated systems. AI systems , including GAI \\nsystems,  can increase the speed and scale at which harmful biases manifest  and are acted upon, \\npotentially  perpetuati ng and amplify ing harms to individuals, groups, communities, organizations, and \\nsociety . For example,  when prompted to generate images of CEOs, doctors, lawyers, and judges, current \\ntext-to-image models underrepresent  women and/or racial minorities , and people with disabilities . \\nImage generator models have also produce d biased or stereotyped output for  various demographic'),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf', 'page': 59, '_id': '6860cf964d8240dcb54e6868b373eb14', '_collection_name': 'Midterm Embedding Eval'}, page_content='National Institute of Standards and Technology (2024) Adversarial Machine Learning: A Taxonomy and \\nTerminology of Attacks and Mitigations https://csrc.nist.gov/pubs/ai/100/2/e2023/final  \\nNational Institute of Standards and Technology  (2023) AI Risk Management Framework . \\nhttps://www.nist.gov/itl/ai -risk-management -framework  \\nNational Institute of Standards and Technology  (2023) AI Risk Management Framework, Chapter 3: AI \\nRisks and Trustworthiness.  \\nhttps://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Foundational_Information/3- sec-characteristics  \\nNational Institute of Standards and Technology  (2023) AI Risk Management Framework , Chapter 6 : AI \\nRMF Profiles. https://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Core_And_Profiles/6 -sec-profile  \\nNational Institute of Standards and Technology  (2023) AI Risk Management Framework, Appendix A: \\nDescriptions of AI Actor Tasks . \\nhttps://airc.nist.gov/AI_RMF_Knowledge_Base/AI_RMF/Appendices/Appendix_A#:~:text=AI%20actors%\\n20in%20this%20category,data%20providers%2C%20system%20funders%2C%20product'),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf', 'page': 12, '_id': '0ad84096cbc64d01b44ef852fc024e0f', '_collection_name': 'Midterm Embedding Eval'}, page_content='There may also be concerns  about  emotional entanglement  between humans and GAI systems, which \\ncould lead to negative psychological impacts . \\nTrustworthy AI Characteristics: Accountable and Transparent, Explainable and Interpretable, Fair with \\nHarmful Bias Managed, Privacy Enhanced, Safe , Valid and Reliable  \\n2.8. Information Integrity  \\nInformation integrity  describes the “ spectrum of information and associated patterns of its creation, \\nexchange, and consumption in society .” High-integrity information can be trusted; “distinguishes fact \\nfrom fiction, opinion, and inference; acknowledges uncertainties; and is transparent about its level of \\nvetting. This information can be linked to the original source(s) with appropriate evidence. High- integrity \\ninformation is also accurate and reliable, can be verified and  authenticated, has a clear chain of custody, \\nand creates reasonable expectations about when its validity  may expire. ”11 \\n \\n \\n11 This definition of information integrity is derived from the  2022 White House Roadmap for Researchers on \\nPriorities Related to Information Integrity Research and Development.'),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 44, '_id': '1354701fd1f14d3b88b2d1fe0d433bb1', '_collection_name': 'Midterm Embedding Eval'}, page_content='generation of artificially intelligent partners.95 The National Science Foundation’s program on Fairness in \\nArtificial Intelligence also includes a specific interest in research foundations for explainable AI.96\\n45'),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/Blueprint-for-an-AI-Bill-of-Rights.pdf', 'page': 21, '_id': 'c5c0419fd2424b0eaa852cd42b94375e', '_collection_name': 'Midterm Embedding Eval'}, page_content=\"SAFE AND EFFECTIVE \\nSYSTEMS \\nHOW THESE PRINCIPLES CAN MOVE INTO PRACTICE\\nReal-life examples of how these principles can become reality, through laws, policies, and practical \\ntechnical and sociotechnical approaches to protecting rights, opportunities, and access. \\nSome U.S government agencies have developed specific frameworks for ethical use of AI \\nsystems. The Department of Energy (DOE) has activated the AI Advancement Council that oversees coordina -\\ntion and advises on implementation of the DOE AI Strategy and addresses issues and/or escalations on the \\nethical use and development of AI systems.20 The Department of Defense has adopted Artificial Intelligence \\nEthical Principles, and tenets for Responsible Artificial Intelligence specifically tailored to its national \\nsecurity and defense activities.21 Similarl y, the U.S. Intelligence Community (IC) has developed the Principles \\nof Artificial Intelligence Ethics for the Intelligence Community to guide personnel on whether and how to \\ndevelop and use AI in furtherance of the IC's mission, as well as an AI Ethics Framework to help implement \\nthese principles.22\\nThe National Science Foundation (NSF) funds extensive research to help foster the \\ndevelopment of automated systems that adhere to and advance their safety, security and \\neffectiveness. Multiple NSF programs support research that directly addresses many of these principles: \\nthe National AI Research Institutes23 support research on all aspects of safe, trustworth y, fai r, and explainable \\nAI algorithms and systems; the Cyber Physical Systems24 program supports research on developing safe \\nautonomous and cyber physical systems with AI components; the Secure and Trustworthy Cyberspace25 \\nprogram supports research on cybersecurity and privacy enhancing technologies in automated systems; the \\nFormal Methods in the Field26 program supports research on rigorous formal verification and analysis of\"),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf', 'page': 35, '_id': '4a03397cb66849518171d94b407075a5', '_collection_name': 'Midterm Embedding Eval'}, page_content='32 MEASURE 2.6:  The AI system is evaluated regularly for safety risks – as identified in the MAP function. The AI system to be \\ndeployed is demonstrated to be safe, its residual negative risk does not exceed the risk tolerance, and it can fail safely, p articularly if \\nmade  to operate beyond its knowledge limits. Safety metrics reflect system reliability and robustness, real- time monitoring, and \\nresponse times for AI system failures.  \\nAction ID  Suggested Action  GAI Risks  \\nMS-2.6-001 Assess adverse impacts , including health and wellbeing impacts for value chain  \\nor other AI Actors  that are exposed to sexually explicit,  offensive , or violent \\ninformation during  GAI training and maintenance.  Human -AI Configuration ; Obscene, \\nDegrading, and/or Abusive \\nContent ; Value Chain and \\nComponent Integration; Dangerous , Violent, or Hateful \\nContent  \\nMS-2.6-002 Assess existence or  levels of harmful bias , intellectual property infringement, \\ndata privacy violations, obscenity, extremism, violence, or CBRN information in \\nsystem training data.  Data Privacy ; Intellectual Property ; \\nObscene, Degrading, and/or Abusive Content ; Harmful Bias and \\nHomogenization ; Dangerous , \\nViolent, or Hateful Content ; CBRN \\nInformation  or Capabilities  \\nMS-2.6-003 Re-evaluate safety features of fine -tuned models when the negative risk exceeds \\norganizational risk tolerance.  Dangerous , Violent, or Hateful \\nContent  \\nMS-2.6-004 Review GAI system outputs for validity and safety: Review generated code to assess risks that may arise from unreliable downstream decision -making.  Value Chain and Component \\nIntegration ; Dangerous , Violent, or \\nHateful Content  \\nMS-2.6-005 Verify that GAI system architecture can monitor outputs and performance, and \\nhandle, recover from, and repair errors when security anomalies, threats and impacts are detected.  Confabulation ; Information \\nIntegrity ; Information Security'),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf', 'page': 34, '_id': '9c2f57bf30814277bf1a20c88ed563be', '_collection_name': 'Midterm Embedding Eval'}, page_content='31 MS-2.3-004 Utilize a purpose -built testing environment such as NIST Dioptra to empirically \\nevaluate GAI trustworthy characteristics.  CBRN Information or Capabilities ; \\nData Privacy ; Confabulation ; \\nInformation Integrity ; Information \\nSecurity ; Dangerous , Violent, or \\nHateful Content ; Harmful Bias and \\nHomogenization  \\nAI Actor Tasks:  AI Deployment, TEVV  \\n \\nMEASURE 2.5:  The AI system to be deployed is demonstrated to be valid and reliable. Limitations of the generalizability beyond the \\nconditions under which the technology was developed are documented.  \\nAction ID  Suggested Action  Risks  \\nMS-2.5-001 Avoid extrapolating GAI system performance or capabilities from narrow, non -\\nsystematic, and anecdotal assessments.  Human -AI Configuration ; \\nConfabulation  \\nMS-2.5-002 Document the extent to which human domain knowledge is employed to \\nimprove GAI system performance, via, e.g., RLHF, fine -tuning, retrieval-\\naugmented generation, content moderation, business rules.  Human -AI Configuration  \\nMS-2.5-003 Review and verify sources and citations in GAI system outputs during pre -\\ndeployment risk measurement and ongoing monitoring activities.  Confabulation  \\nMS-2.5-004 Track and document instances of anthropomorphization  (e.g., human images, \\nmentions of human feelings, cyborg imagery or motifs) in GAI system interfaces.  Human -AI Configuration  \\nMS-2.5-0 05 Verify GAI system training data and TEVV data provenance, and that fine -tuning  \\nor retrieval- augmented generation data is grounded.  Information Integrity  \\nMS-2.5-0 06 Regularly review security and safety guardrails, especially if the GAI system is \\nbeing operated in novel circumstances. This includes reviewing reasons why the \\nGAI system was initially assessed as being safe to deploy.  Information Security ; Dangerous , \\nViolent, or Hateful  Content  \\nAI Actor Tasks:  Domain Experts, TEVV'),\n",
+       "  Document(metadata={'source': '/Users/xico/AIMakerSpace-Midterm/AI_Risk_Management_Framework.pdf', 'page': 41, '_id': '318ce3f5be254529966a22fbe12f9e13', '_collection_name': 'Midterm Embedding Eval'}, page_content='Information Integrity  \\nMS-3.3-003 Evaluate potential biases and stereotypes that could emerge from the AI -\\ngenerated content using appropriate methodologies including computational testing methods as well as evaluating structured feedback input.  Harmful Bias and Homogenization')]}"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "multiquery_ft_embedding_rag_chain.invoke({\"question\": \"What are Trustworthy AI Characteristics?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "answers = []\n",
+    "contexts = []\n",
+    "\n",
+    "for question in test_questions:\n",
+    "  response = multiquery_ft_embedding_rag_chain.invoke({\"question\" : question})\n",
+    "  answers.append(response[\"answer\"])\n",
+    "  contexts.append([context.page_content for context in response[\"source_documents\"]])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import Dataset\n",
+    "\n",
+    "multiquery_ft_embedding_dataset = Dataset.from_dict({\n",
+    "    \"question\" : test_questions,\n",
+    "    \"answer\" : answers,\n",
+    "    \"contexts\" : contexts,\n",
+    "    \"ground_truth\" : test_groundtruths\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'question': 'What is the significance of providing notice and explanation as a legal requirement in the context of automated systems?',\n",
+       " 'answer': \"Providing notice and explanation as a legal requirement in the context of automated systems is significant for several reasons:\\n\\n1. **Transparency**: It ensures that individuals are aware when automated systems are being used to make decisions that affect them. This transparency is crucial for building trust between the public and the entities deploying these systems.\\n\\n2. **Accountability**: By clearly identifying the entities responsible for the design and use of automated systems, it holds them accountable for the decisions made by these systems. This accountability is essential for addressing any potential harms or errors that may arise from automated decision-making.\\n\\n3. **Informed Decision-Making**: Notice and explanation allow individuals to understand how and why decisions are made, which is vital for them to contest or appeal those decisions if necessary. For example, if someone is denied a loan or a job, knowing the criteria used by the automated system can help them address any inaccuracies or biases.\\n\\n4. **Protection of Rights**: Automated systems can significantly impact people's lives, from employment opportunities to legal outcomes. Providing notice and explanations helps protect individuals' rights by ensuring they have the information needed to understand and challenge decisions that may adversely affect them.\\n\\n5. **Encouraging Fairness**: When individuals are informed about the use of automated systems and the rationale behind decisions, it can help mitigate biases and promote fairness. This is particularly important in sensitive areas like criminal justice, healthcare, and employment.\\n\\n6. **Facilitating Oversight**: Clear notice and explanations enable oversight bodies, researchers, and the public to scrutinize automated systems, ensuring they operate fairly and effectively. This oversight is crucial for identifying and correcting any issues that may arise.\\n\\nIn summary, the legal requirement for notice and explanation in automated systems is essential for ensuring transparency, accountability, and fairness, ultimately protecting individuals' rights and fostering trust in these technologies (Source: [document name], p. [specific page number not provided]).\",\n",
+       " 'contexts': ['NOTICE & \\nEXPLANATION \\nWHAT SHOULD BE EXPECTED OF AUTOMATED SYSTEMS\\nThe expectations for automated systems are meant to serve as a blueprint for the development of additional \\ntechnical standards and practices that are tailored for particular sectors and contexts. \\nAn automated system should provide demonstrably clear, timely, understandable, and accessible notice of use, and \\nexplanations as to how and why a decision was made or an action was taken by the system. These expectations are explained below. \\nProvide clear, timely, understandable, and accessible notice of use and explanations \\nGenerally accessible plain language documentation. The entity responsible for using the automated \\nsystem should ensure that documentation describing the overall system (including any human components) is \\npublic and easy to find. The documentation should describe, in plain language, how the system works and how \\nany automated component is used to determine an action or decision. It should also include expectations about \\nreporting described throughout this framework, such as the algorithmic impact assessments described as \\npart of Algorithmic Discrimination Protections. \\nAccount able. Notices should clearly identify the entity r esponsible for designing each component of the \\nsystem and the entity using it. \\nTimely and up-to-date. Users should receive notice of the use of automated systems in advance of using or \\nwhile being impacted by the technolog y. An explanation should be available with the decision itself, or soon \\nthereafte r. Notice should be kept up-to-date and people impacted by the system should be notified of use case \\nor key functionality changes. \\nBrief and clear. Notices and explanations should be assessed, such as by research on users’ experiences, \\nincluding user testing, to ensure that the people using or impacted by the automated system are able to easily',\n",
+       "  'NOTICE & \\nEXPLANATION \\nWHY THIS PRINCIPLE IS IMPORTANT\\nThis section provides a brief summary of the problems which the principle seeks to address and protect \\nagainst, including illustrative examples. \\nAutomated systems now determine opportunities, from employment to credit, and directly shape the American \\npublic’s experiences, from the courtroom to online classrooms, in ways that profoundly impact people’s lives. But this expansive impact is not always visible. An applicant might not know whether a person rejected their resume or a hiring algorithm moved them to the bottom of the list. A defendant in the courtroom might not know if a judge deny\\n-\\ning their bail is informed by an automated system that labeled them “high risk.” From correcting errors to contesting decisions, people are often denied the knowledge they need to address the impact of automated systems on their lives. Notice and explanations also serve an important safety and efficacy purpose, allowing experts to verify the reasonable\\n-\\nness of a recommendation before enacting it. \\nIn order to guard against potential harms, the American public needs to know if an automated system is being used. Clear, brief, and understandable notice is a prerequisite for achieving the other protections in this framework. Like\\n-\\nwise, the public is often unable to ascertain how or why an automated system has made a decision or contributed to a particular outcome. The decision-making processes of automated systems tend to be opaque, complex, and, therefore, unaccountable, whether by design or by omission. These factors can make explanations both more challenging and more important, and should not be used as a pretext to avoid explaining important decisions to the people impacted by those choices. In the context of automated systems, clear and valid explanations should be recognized as a baseline requirement.',\n",
+       "  'or label to ensure the goal of the automated system is appropriately identified and measured. Additionally , \\njustification should be documented for each data attribute and source to explain why it is appropriate to use \\nthat data to inform the results of the automated system and why such use will not violate any applicable laws. \\nIn cases of high-dimensional and/or derived attributes, such justifications can be provided as overall \\ndescriptions of the attribute generation process and appropriateness. \\n19',\n",
+       "  'Meaningful access to examine the system. Designers, developers, and deployers of automated \\nsystems should consider limited waivers of confidentiality (including those related to trade secrets) where necessary in order to provide meaningful oversight of systems used in sensitive domains, incorporating mea\\n-\\nsures to protect intellectual property and trade secrets from unwarranted disclosure as appropriate. This includes (potentially private and protected) meaningful access to source code, documentation, and related data during any associated legal discovery, subject to effective confidentiality or court orders. Such meaning\\n-\\nful access should include (but is not limited to) adhering to the principle on Notice and Explanation using the highest level of risk so the system is designed with built-in explanations; such systems should use fully-trans\\n-\\nparent models where the model itself can be understood by people needing to directly examine it. \\nDemonstrate access to human alternatives, consideration, and fallback \\nReporting. Reporting should include an assessment of timeliness and the extent of additional burden for human alternatives, aggregate statistics about who chooses the human alternative, along with the results of the assessment about brevity, clarity, and accessibility of notice and opt-out instructions. Reporting on the accessibility, timeliness, and effectiveness of human consideration and fallback should be made public at regu\\n-',\n",
+       "  \"Providing notice has long been a standard practice, and in many cases is a legal requirement, when, for example, making a video recording of someone (outside of a law enforcement or national security context). In some cases, such as credit, lenders are required to provide notice and explanation to consumers. Techniques used to automate the process of explaining such systems are under active research and improvement and such explanations can take many forms. Innovative companies and researchers are rising to the challenge and creating and deploying explanatory systems that can help the public better understand decisions that impact them. \\nWhile notice and explanation requirements are already in place in some sectors or situations, the American public deserve to know consistently and across sectors if an automated system is being used in a way that impacts their rights, opportunities, or access. This knowledge should provide confidence in how the public is being treated, and trust in the validity and reasonable use of automated systems. \\n• A lawyer representing an older client with disabilities who had been cut off from Medicaid-funded home\\nhealth-care assistance couldn't determine why\\n, especially since the decision went against historical access\\npractices. In a court hearing, the lawyer learned from a witness that the state in which the older client\\nlived \\nhad recently adopted a new algorithm to determine eligibility.83 The lack of a timely explanation made it\\nharder \\nto understand and contest the decision.\\n•\\nA formal child welfare investigation is opened against a parent based on an algorithm and without the parent\\never \\nbeing notified that data was being collected and used as part of an algorithmic child maltreatment\\nrisk assessment.84 The lack of notice or an explanation makes it harder for those performing child\\nmaltreatment assessments to validate the risk assessment and denies parents knowledge that could help them\\ncontest a decision.\\n41\",\n",
+       "  \"find notices and explanations, read them quickl y, and understand and act on them. This includes ensuring that \\nnotices and explanations are accessible to users with disabilities and are available in the language(s) and read-\\ning level appropriate for the audience. Notices and explanations may need to be available in multiple forms, \\n(e.g., on pape r, on a physical sign, or online), in order to meet these expectations and to be accessible to the \\nAmerican public. \\nProvide explanations as to how and why a decision was made or an action was taken by an \\nautomated system \\nTailored to the purpose. Explanations should be tailored to the specific purpose for which the user is \\nexpected to use the explanation, and should clearly state that purpose. An informational explanation might differ from an explanation provided to allow for the possibility of recourse, an appeal, or one provided in the context of a dispute or contestation process. For the purposes of this framework, 'explanation' should be construed broadly. An explanation need not be a plain-language statement about causality but could consist of any mechanism that allows the recipient to build the necessary understanding and intuitions to achieve the stated purpose. Tailoring should be assessed (e.g., via user experience research). \\nTailored to the target of the explanation. Explanations should be targeted to specific audiences and clearly state that audience. An explanation provided to the subject of a decision might differ from one provided to an advocate, or to a domain expert or decision maker. Tailoring should be assessed (e.g., via user experience research). \\n43\"],\n",
+       " 'ground_truth': 'Providing notice and explanation as a legal requirement in the context of automated systems is significant because it allows individuals to understand how automated systems are impacting their lives. It helps in correcting errors, contesting decisions, and verifying the reasonableness of recommendations before enacting them. Clear and valid explanations are essential to ensure transparency, accountability, and trust in the use of automated systems across various sectors.'}"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "multiquery_ft_embedding_dataset[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ragas import evaluate\n",
+    "from ragas.metrics import (\n",
+    "    faithfulness,\n",
+    "    answer_relevancy,\n",
+    "    answer_correctness,\n",
+    "    context_recall,\n",
+    "    context_precision,\n",
+    ")\n",
+    "\n",
+    "metrics = [\n",
+    "    faithfulness,\n",
+    "    answer_relevancy,\n",
+    "    context_recall,\n",
+    "    context_precision,\n",
+    "    answer_correctness,\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "190157ee020d4f2191f04a2682ba8737",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Evaluating:   0%|          | 0/120 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "multiquery_ft_embedding_results = evaluate(multiquery_ft_embedding_dataset, metrics)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'faithfulness': 0.8684, 'answer_relevancy': 0.9558, 'context_recall': 0.9444, 'context_precision': 0.9537, 'answer_correctness': 0.6034}"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "multiquery_ft_embedding_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "multiquery_ft_embedding_results_df = multiquery_ft_embedding_results.to_pandas()\n",
+    "multiquery_ft_embedding_results_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "multiquery_ft_embedding_results_df.to_csv(\"multiquery_ft_embedding_ragas_results.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "multiquery_ft_embedding_metrics_df = pd.DataFrame(list(multiquery_ft_embedding_results.items()), columns=['Metric', 'Fine-Tune Embedding'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Metric</th>\n",
+       "      <th>Fine-Tune Embedding</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>faithfulness</td>\n",
+       "      <td>0.868351</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>answer_relevancy</td>\n",
+       "      <td>0.955777</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>context_recall</td>\n",
+       "      <td>0.944444</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>context_precision</td>\n",
+       "      <td>0.953668</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>answer_correctness</td>\n",
+       "      <td>0.603407</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "               Metric  Fine-Tune Embedding\n",
+       "0        faithfulness             0.868351\n",
+       "1    answer_relevancy             0.955777\n",
+       "2      context_recall             0.944444\n",
+       "3   context_precision             0.953668\n",
+       "4  answer_correctness             0.603407"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "multiquery_ft_embedding_metrics_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "multiquery_ft_embedding_metrics_df.to_csv(\"multiquery_ft_embedding_metrics.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Metric</th>\n",
+       "      <th>Baseline</th>\n",
+       "      <th>Fine-Tune Embedding</th>\n",
+       "      <th>Baseline -&gt; Fine-Tune Embedding</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>faithfulness</td>\n",
+       "      <td>0.895359</td>\n",
+       "      <td>0.868351</td>\n",
+       "      <td>-0.027007</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>answer_relevancy</td>\n",
+       "      <td>0.955419</td>\n",
+       "      <td>0.955777</td>\n",
+       "      <td>0.000358</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>context_recall</td>\n",
+       "      <td>0.934028</td>\n",
+       "      <td>0.944444</td>\n",
+       "      <td>0.010417</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>context_precision</td>\n",
+       "      <td>0.937500</td>\n",
+       "      <td>0.953668</td>\n",
+       "      <td>0.016168</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>answer_correctness</td>\n",
+       "      <td>0.629267</td>\n",
+       "      <td>0.603407</td>\n",
+       "      <td>-0.025861</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "               Metric  Baseline  Fine-Tune Embedding  \\\n",
+       "0        faithfulness  0.895359             0.868351   \n",
+       "1    answer_relevancy  0.955419             0.955777   \n",
+       "2      context_recall  0.934028             0.944444   \n",
+       "3   context_precision  0.937500             0.953668   \n",
+       "4  answer_correctness  0.629267             0.603407   \n",
+       "\n",
+       "   Baseline -> Fine-Tune Embedding  \n",
+       "0                        -0.027007  \n",
+       "1                         0.000358  \n",
+       "2                         0.010417  \n",
+       "3                         0.016168  \n",
+       "4                        -0.025861  "
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_baseline_ft_embeddings = pd.merge(baseline_metrics, multiquery_ft_embedding_metrics_df, on='Metric')\n",
+    "\n",
+    "df_baseline_ft_embeddings['Baseline -> Fine-Tune Embedding'] = df_baseline_ft_embeddings['Fine-Tune Embedding'] - df_baseline_ft_embeddings['Baseline']\n",
+    "\n",
+    "df_baseline_ft_embeddings"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

app.py

@@ -33,8 +33,8 @@ else:
         documents.extend(loader.load())
 
     text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=2000,
-        chunk_overlap=0,
+        chunk_size=1000,
+        chunk_overlap=200,
     )
 
     docs = text_splitter.split_documents(documents)

contextual_compression_metrics.csv

@@ -0,0 +1,6 @@
+Metric,ContextualCompression
+faithfulness,0.7490919748849417
+answer_relevancy,0.9139929840714447
+context_recall,0.7256944444444443
+context_precision,0.9050925925582561
+answer_correctness,0.5706849416579475

multiquery_ft_embedding_metrics.csv

@@ -0,0 +1,6 @@
+Metric,Fine-Tune Embedding
+faithfulness,0.8683514087024626
+answer_relevancy,0.955776890181432
+context_recall,0.9444444444444443
+context_precision,0.9536680366355096
+answer_correctness,0.6034069059659971

multiquery_metrics.csv

@@ -0,0 +1,6 @@
+Metric,MultiQuery
+faithfulness,0.8968038152225203
+answer_relevancy,0.9532110694802333
+context_recall,0.890625
+context_precision,0.9207324735259621
+answer_correctness,0.690058377538071