Adding application, not finished yet, still wont expand text

app.py

@@ -0,0 +1,130 @@
+from time import perf_counter
+
+from jinja2 import Template
+
+from backend.semantic_search import qd_retriever
+
+template_string = """
+Instructions: Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+Context:
+---
+{% for doc in documents %}
+    {{ doc.content }}
+    ---
+{% endfor %}
+Query: {{ query }}
+"""
+
+md_template_string = """
+<b>Instructions</b>: 
+<span style="color: green;">Use the following pieces of context to answer the question at the end.<br>If you don't know the answer, just say that you don't know, <span style="color: green; font-weight: bold;">don't try to make up an answer.</span></span><br>
+
+<b>Context</b>:
+{% for doc in documents %}
+    <div id=\"box{{ loop.index }}\" style=\"border: 2px solid #aaa; padding: 10px; margin-top: 10px; border-radius: 5px; background-color: #1E90FF; position: relative; cursor: pointer;\">
+        <div style=\"font-size: 0.8em; position: absolute; top: 10px; left: 10px;\"><b>Doc {{ loop.index }}</b></div>
+        <span id=\"doc{{ loop.index }}-short\" style=\"color: white; display: block; margin-top: 20px;\">{{ doc.content[:50] }}...</span>
+        <span id=\"doc{{ loop.index }}-full\" style=\"color: white; display: none; margin-top: 20px;\">{{ doc.content }}</span>
+    </div>
+{% endfor %}
+<b>Query</b>: <span style=\"color: yellow;\">{{ query }}</span>
+<script>
+document.addEventListener("DOMContentLoaded", function() {
+    {% for doc in documents %}
+        document.getElementById("box{{ loop.index }}").addEventListener("click", function() {
+            toggleContent('doc{{ loop.index }}');
+        });
+    {% endfor %}
+});
+
+function toggleContent(docID) {
+    var shortContent = document.getElementById(docID + '-short');
+    var fullContent = document.getElementById(docID + '-full');
+    if (fullContent.style.display === 'none') {
+        shortContent.style.display = 'none';
+        fullContent.style.display = 'block';
+    } else {
+        shortContent.style.display = 'block';
+        fullContent.style.display = 'none';
+    }
+}
+</script>
+"""
+
+template = Template(template_string)
+md_template = Template(md_template_string)
+import gradio as gr
+
+from backend.query_llm import generate
+
+
+def add_text(history, text):
+    history = [] if history is None else history
+    history = history + [(text, None)]
+    return history, gr.Textbox(value="", interactive=False)
+
+
+def bot(history, system_prompt=""):
+    top_k = 5
+    query = history[-1][0]
+
+    # Retrieve documents relevant to query
+    document_start = perf_counter()
+    documents = qd_retriever.retrieve(query, top_k=top_k)
+    document_time = document_start - perf_counter()
+
+    # Create Prompt
+    prompt = template.render(documents=documents, query=query)
+    md_prompt = md_template.render(documents=documents, query=query)
+
+    # # Query LLM with prompt based on relevant documents
+    # llm_start = perf_counter()
+    # result = generate(prompt=prompt, history='')
+    # llm_time = llm_start - perf_counter()
+    # times = (document_time, llm_time)
+
+    history[-1][1] = ""
+    for character in generate(prompt, history[:-1]):
+        history[-1][1] = character
+        yield history, md_prompt
+
+
+with gr.Blocks() as demo:
+    with gr.Tab("Application"):
+        chatbot = gr.Chatbot(
+                [],
+                elem_id="chatbot",
+                avatar_images=('examples/lama.jpeg', 'examples/lama2.jpeg'),
+                bubble_full_width=False,
+                show_copy_button=True,
+                show_share_button=True,
+                )
+
+        with gr.Row():
+            txt = gr.Textbox(
+                    scale=3,
+                    show_label=False,
+                    placeholder="Enter text and press enter",
+                    container=False,
+                    )
+            txt_btn = gr.Button(value="Submit text", scale=1)
+
+        prompt_md = gr.HTML()
+        # Turn off interactivity while generating if you hit enter
+        txt_msg = txt_btn.click(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
+                bot, chatbot, [chatbot, prompt_md])
+
+        # Turn it back on
+        txt_msg.then(lambda: gr.Textbox(interactive=True), None, [txt], queue=False)
+
+        # Turn off interactivity while generating if you hit enter
+        txt_msg = txt.submit(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
+                bot, chatbot, [chatbot, prompt_md])
+
+        # Turn it back on
+        txt_msg.then(lambda: gr.Textbox(interactive=True), None, [txt], queue=False)
+
+    gr.Examples(['What is the largest city on earth?', 'Who has the record for the fastest mile?'], txt)
+
+demo.queue()
+demo.launch(debug=True)

backend/query_llm.py

@@ -0,0 +1,86 @@
+from typing import Any, Dict, Generator, List
+
+import gradio as gr
+from huggingface_hub import InferenceClient
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
+
+temperature = 0.9
+top_p = 0.6
+repetition_penalty = 1.2
+
+text_client = InferenceClient(
+        "mistralai/Mistral-7B-Instruct-v0.1"
+        )
+
+
+def format_prompt(message: str) -> str:
+    """
+    Formats the given message using a chat template.
+
+    Args:
+        message (str): The user message to be formatted.
+
+    Returns:
+        str: Formatted message after applying the chat template.
+    """
+
+    # Create a list of message dictionaries with role and content
+    messages: List[Dict[str, Any]] = [{'role': 'user', 'content': message}]
+
+    # Return the message after applying the chat template
+    return tokenizer.apply_chat_template(messages, tokenize=False)
+
+
+def generate(prompt: str, history: str, temperature: float = 0.9, max_new_tokens: int = 256,
+             top_p: float = 0.95, repetition_penalty: float = 1.0) -> Generator[str, None, str]:
+    """
+    Generate a sequence of tokens based on a given prompt and history using Mistral client.
+
+    Args:
+        prompt (str): The initial prompt for the text generation.
+        history (str): Context or history for the text generation.
+        temperature (float, optional): The softmax temperature for sampling. Defaults to 0.9.
+        max_new_tokens (int, optional): Maximum number of tokens to be generated. Defaults to 256.
+        top_p (float, optional): Nucleus sampling probability. Defaults to 0.95.
+        repetition_penalty (float, optional): Penalty for repeated tokens. Defaults to 1.0.
+
+    Returns:
+        Generator[str, None, str]: A generator yielding chunks of generated text.
+                                   Returns a final string if an error occurs.
+    """
+
+    temperature = max(float(temperature), 1e-2)  # Ensure temperature isn't too low
+    top_p = float(top_p)
+
+    generate_kwargs = {
+        'temperature': temperature,
+        'max_new_tokens': max_new_tokens,
+        'top_p': top_p,
+        'repetition_penalty': repetition_penalty,
+        'do_sample': True,
+        'seed': 42,
+        }
+
+    formatted_prompt = format_prompt(prompt)
+
+    try:
+        stream = text_client.text_generation(formatted_prompt, **generate_kwargs,
+                                             stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield output
+
+    except Exception as e:
+        if "Too Many Requests" in str(e):
+            print("ERROR: Too many requests on Mistral client")
+            gr.Warning("Unfortunately Mistral is unable to process")
+            return "Unfortunately, I am not able to process your request now."
+        else:
+            print("Unhandled Exception:", str(e))
+            gr.Warning("Unfortunately Mistral is unable to process")
+            return "I do not know what happened, but I couldn't understand you."
+
+    return output

backend/retrieval_augmented_generation.py

@@ -0,0 +1,38 @@
+from time import perf_counter
+from jinja2 import Template
+
+from backend.query_llm import generate
+from backend.semantic_search import qd_retriever
+
+template_string = """
+Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+Context:
+---
+{% for doc in documents %}
+    {{ doc.content }}
+    ---
+{% endfor %}
+Query: {{ query }}
+"""
+
+template = Template(template_string)
+
+
+def rag(query, top_k=5):
+
+    # Retrieve documents relevant to query
+    document_start = perf_counter()
+    documents = qd_retriever.retrieve(query, top_k=top_k)
+    document_time = document_start - perf_counter()
+
+    # Create Prompt
+    prompt = template.render(documents=documents, query=query)
+
+    # Query LLM with prompt based on relevant documents
+    llm_start = perf_counter()
+    result = generate(prompt=prompt, history='')
+    llm_time = llm_start - perf_counter()
+
+    times = (document_time, llm_time)
+    return prompt, result
+

backend/semantic_search.py

@@ -0,0 +1,10 @@
+from qdrant_haystack import QdrantDocumentStore
+from haystack.nodes import EmbeddingRetriever
+from pathlib import Path
+
+proj_dir = Path(__file__).parents[1]
+qd_document_store = QdrantDocumentStore(path=str(proj_dir/'Qdrant'), index='RAGDemo')
+qd_retriever = EmbeddingRetriever(document_store=qd_document_store,
+                                  embedding_model="BAAI/bge-base-en-v1.5",
+                                  model_format="sentence_transformers",
+                                  use_gpu=False)

utilities/retrievers.py

@@ -0,0 +1,26 @@
+import pickle
+from logging import getLogger
+from pathlib import Path
+
+import torch
+from haystack.nodes import EmbeddingRetriever
+from qdrant_haystack import QdrantDocumentStore
+
+logger = getLogger(__name__)
+
+proj_dir = Path(__file__).parents[1]
+
+st_document_store_path = proj_dir / 'haystack_pickles' / 'simple-wiki_all-mpnet-base-v2_document-store.pkl'
+
+logger.warning('Loading Document Store...')
+with open(st_document_store_path, 'rb') as handle:
+    st_document_store = pickle.load(handle)
+logger.warning('Loaded Document Store...')
+
+qd_document_store = QdrantDocumentStore(path=str(proj_dir/'Qdrant'))
+
+qd_document_store.main_device = torch.device('cpu')
+qd_retriever = EmbeddingRetriever(document_store=qd_document_store,
+                                  embedding_model="sentence-transformers/all-mpnet-base-v2",
+                                  model_format="sentence_transformers",
+                                  use_gpu=True)