Add vector database

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.md filter=lfs diff=lfs merge=lfs -text
+*.py filter=lfs diff=lfs merge=lfs -text
+*.sqlite3 filter=lfs diff=lfs merge=lfs -text
+*.txt filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,3 @@
----
-title: QuerySummarizer
-emoji: 🏢
-colorFrom: purple
-colorTo: gray
-sdk: streamlit
-sdk_version: 1.33.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+version https://git-lfs.github.com/spec/v1
+oid sha256:58993ad738efab98d5036aa7ac4ced3eef97a74f064ee15c44f7c7ad766e6fa2
+size 239

app.py

@@ -1,308 +1,3 @@
-import streamlit as st
-
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from transformers.utils import is_flash_attn_2_available
-from transformers import BitsAndBytesConfig
-import pandas as pd
-import os
-import torch
-import numpy as np
-from scipy import sparse
-from sklearn.metrics.pairwise import cosine_similarity
-from scipy import sparse
-from langchain_community.vectorstores import Chroma
-from langchain_community.embeddings.sentence_transformer import (
-    SentenceTransformerEmbeddings,
-)
-
-# SET TO WIDE LAYOUT
-st.set_page_config(layout="wide")
-
-#_______________________________________________SET VARIABLES_____________________________________________________
-
-MODEL_ID = 'google/gemma-2b-it'
-CHUNK_SIZE = 1000
-OVERLAP_SIZE = 100
-EMBEDDING = "all-MiniLM-L6-v2"
-COLLECTION_NAME = f'vb_summarizer_{EMBEDDING}_test'
-CHROMA_DATA_PATH = 'feedback_360'
-
-#_______________________________________________LOAD MODELS_____________________________________________________
-# LOAD MODEL
-@st.cache_resource
-def load_model(model_id) :
-
-    HF_TOKEN = os.environ['HF_TOKEN']
-    print(torch.backends.mps.is_available())
-    #device = torch.device("mps") if torch.backends.mps.is_available() else "cpu"
-    device = 'cpu'
-    print(device)
-
-    if device=='cpu' :
-        print('Warning! No GPU available')
-
-    # IMPORT MODEL
-    
-    print(model_id)
-
-    quantization_config = BitsAndBytesConfig(load_in_4bit=True,
-                                            bnb_4bit_compute_dtype=torch.float16)
-
-    # if (is_flash_attn_2_available()) and (torch.cuda.get_device_capability(0)[0] >= 8):
-    #   attn_implementation = "flash_attention_2"
-    # else:
-    #   attn_implementation = "sdpa"
-    # print(f"[INFO] Using attention implementation: {attn_implementation}")
-
-    tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model_id, token=HF_TOKEN)
-
-    llm_model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path=model_id,
-                                                    token=HF_TOKEN,
-                                                    torch_dtype=torch.float16,
-                                                    #quantization_config=quantization_config if quantization_config else None,
-                                                    low_cpu_mem_usage=False,) # use full memory
-                                                    #attn_implementation=attn_implementation) # which attention version to use
-    llm_model.to(device)
-    return llm_model, tokenizer, device
-
-# LOAD VECTORSTORE
-@st.cache_resource
-def load_data(embedding) :
-    # CREATE EMBEDDING
-    embedding_function = SentenceTransformerEmbeddings(model_name=embedding)
-    db3 = Chroma(collection_name = COLLECTION_NAME, persist_directory="./chroma", embedding_function = embedding_function)
-    return db3
-
-
-# Create a text element and let the reader know the data is loading.
-model_load_state = st.text('Loading model...')
-# Load 10,000 rows of data into the dataframe.
-llm_model, tokenizer, device = load_model(MODEL_ID)
-# Notify the reader that the data was successfully loaded.
-model_load_state.text('Loading model...done!')
-
-# Create a text element and let the reader know the data is loading.
-data_load_state = st.text('Loading data...')
-# Load 10,000 rows of data into the dataframe.
-vectorstore = load_data(EMBEDDING)
-# Notify the reader that the data was successfully loaded.
-data_load_state.text('Loading data...done!')
-
-
-#_______________________________________________SUMMARIZATION_____________________________________________________
-# INFERENCE
-# def prompt_formatter(reviews, type_of_doc):
-#     return f"""You are a summarization bot.
-#     You will receive {type_of_doc} and you will extract all relevant information from {type_of_doc} and return one paragraph in which you will summarize what was said.
-#     {type_of_doc} are listed below under inputs.
-#     Inputs: {reviews}
-#     Answer :
-#     """
-# def prompt_formatter(reviews, type_of_doc):
-#     return f"""You are a summarization bot.
-#     You will receive {type_of_doc} and you will summarize what was said in the input.
-#     {type_of_doc} are listed below under inputs.
-#     Inputs: {reviews}
-#     Answer :
-#     """
-def prompt_formatter(reviews):
-    return f"""You are a summarization bot.
-    You will receive reviews of Clockify from different users.
-    You will summarize what these reviews said while keeping the information about each of the user.
-    Reviews are listed below.
-    Reviews: {reviews}
-    Answer :
-    """
-
-def mirror_mirror(inputs, prompt_formatter, tokenizer):
-    print('Mirror_mirror')
-    prompt = prompt_formatter(inputs)
-    input_ids = tokenizer(prompt, return_tensors="pt").to(device)
-    outputs = llm_model.generate(**input_ids,
-                                 temperature=0.3,
-                                 do_sample=True,
-                                 max_new_tokens=275)
-    output_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    return prompt, output_text.replace(prompt, '')
-
-
-def summarization(example : str, results_df : pd.DataFrame = pd.DataFrame()) -> pd.DataFrame :
-    
-    # INFERENCE
-    results = []
-    for cnt in range(0,2) : 
-
-        prompt, result = mirror_mirror(example, prompt_formatter, tokenizer)
-        list_temp = [result, example]
-        tokenized = tokenizer(list_temp, return_tensors="pt", padding = True)
-        A = tokenized.input_ids.numpy()
-        A = sparse.csr_matrix(A)
-        score = cosine_similarity(A)[0,1]
-        #print(cosine_similarity(A)[0,1])
-        #print(cosine_similarity(A)[1,0])
-        print(score)
-        if score>0.1 :
-            fin_result = result
-            max_score = score
-            break
-
-        results.append(result)
-        #print(result+'\n\n')
-
-    # tokenize results and example together
-    try  :
-        fin_result 
-    except :
-    # if fin_result not already defined, use the best of available results
-        # add example to results so tokenization is done together (due to padding limitations)
-        results.append(example)
-        tokenized = tokenizer(results, return_tensors="pt", padding = True)
-        A = tokenized.input_ids.numpy()
-        A = sparse.csr_matrix(A)
-        # calculate cosine similarity of each pair 
-        # keep only example X result column
-        scores = cosine_similarity(A)[:,2]
-        # final result is the one with greaters cos_score
-        fin_result = results[np.argmax(scores)]
-        max_score = max(scores)
-
-    #print(fin_result)
-    # save final result and its attributes
-    row = pd.DataFrame({'model' : MODEL_ID, 'prompt' : prompt, 'reviews' : example, 'summarization' : fin_result, 'score' : [max_score] })
-    results_df = pd.concat([results_df,row], ignore_index = True)
-
-    return results_df
-
-def create_filter(group:str=None, platform:str=None, ReviewerPosition:str=None, Industry:str=None, CompanySize:str=None,
-       UsagePeriod:str=None, LinkedinVerified:str=None, Date:str=None, Rating:str=None) :
-    keys = ['group', 'Platform', 'ReviewerPosition', 'Industry', 'CompanySize',
-            'UsagePeriod', 'LinkedinVerified', 'Date', 'Rating']
-    input_keys = [group,platform, ReviewerPosition, Industry, CompanySize, UsagePeriod, LinkedinVerified, Date, Rating]
-    
-    # create filter dict 
-    filter_dict = {}
-    for key, in_key in zip(keys, input_keys) :
-        if not in_key == None and not in_key == ' ':
-            filter_dict[key] = {'$eq' : in_key}
-
-    print(filter_dict)
-    return filter_dict
-
-#_______________________________________________UI_____________________________________________________
-
-st.title("Mirror, mirror, on the cloud, what do Clockify users say aloud?")
-st.subheader("--Clockify review summarizer--")
-
-col1, col2, col3 = st.columns(3, gap = 'small')
-
-with col1:
-   platform = st.selectbox(label = 'Platform',
-             options = [' ', 'Capterra', 'Chrome Extension', 'GetApp', 'AppStore', 'GooglePlay',
-                        'Firefox Extension', 'JIRA Plugin', 'Trustpilot', 'G2',
-                        'TrustRadius']
-             )
-
-with col2:
-   company_size = st.selectbox(label = 'Company Size',
-             options = [' ', '1-10 employees', 'Self-employed', 'self-employed',
-                        'Small-Business(50 or fewer emp.)', '51-200 employees',
-                        'Mid-Market(51-1000 emp.)', '11-50 employees',
-                        '501-1,000 employees', '10,001+ employees', '201-500 employees',
-                        '1,001-5,000 employees', '5,001-10,000 employees',
-                        'Enterprise(> 1000 emp.)', 'Unknown', '1001-5000 employees']
-             )
-
-with col3:
-   linkedin_verified = st.selectbox(label = 'Linkedin Verified',
-             options = [' ', 'True', 'False'],
-             placeholder = 'Choose an option'
-             )
-
-num_to_return = int(st.number_input(label = 'Number of documents to return', min_value = 2, max_value = 50, step = 1))
-
-# group = st.selectbox(label = 'Review Platform Group',
-#              options = ['Software Review Platforms', 'Browser Extension Stores', 'Mobile App Stores', 'Plugin Marketplace']
-#              )
-
-
-
-default_value = "Clockify"
-
-query = st.text_area("Query", default_value, height = 50)
-#type_of_doc = st.text_area("Type of text", 'text', height = 25)
-
-# result = ''
-# score = ''
-# reviews = ''
-
-if 'result' not in st.session_state:
-    st.session_state['result'] = ''
-
-if 'score' not in st.session_state:
-    st.session_state['score'] = ''
-
-if 'reviews' not in st.session_state:
-    st.session_state['reviews'] = ''
-
-col11, col21  = st.columns(2, gap = 'small')
-
-with col11:
-   button_query = st.button('Conquer and query!')
-with col21:
-    button_summarize = st.button('Summon the summarizer!')
-
-
-if  button_query :
-    print('Querying')
-    # create filter from drop-downs
-    filter_dict = create_filter(#group = group,
-                                platform = platform,
-                                CompanySize = company_size,
-                                LinkedinVerified = linkedin_verified
-                    )
-    # FILTER BY META
-    if filter_dict == {} :
-        retriever = vectorstore.as_retriever(search_kwargs = {"k": num_to_return})
-
-    elif len(filter_dict.keys()) == 1 :
-        retriever = vectorstore.as_retriever(search_kwargs = {"k": num_to_return,
-                                                              "filter":  filter_dict})
-    else :
-        retriever = vectorstore.as_retriever(search_kwargs = {"k": num_to_return,
-                                                            "filter":{'$and': [{key : value} for key,value in filter_dict.items()]}
-                                                        }
-        )
-
-    reviews = retriever.get_relevant_documents(query = query)
-    # only get page content
-    st.session_state['reviews'] = [review.page_content for review in reviews]
-    print(st.session_state['reviews'])
-    result = 'You may summarize now!'
-
-if button_summarize :
-    print('Summarization in progress')
-    st.session_state['result'] = 'Summarization in progress'
-    results_df = summarization("\n".join(st.session_state['reviews']))
-    # only one input
-    st.session_state['result'] = results_df.summarization[0]
-    score = results_df.score[0]
-
-
-col12, col22  = st.columns(2, gap = 'small')
-
-with col12:
-   chosen_reviews = st.text_area("Reviews to be summarized", "\n".join(st.session_state['reviews']), height = 275)
-with col22:
-    summarized_text = st.text_area("Summarized text", st.session_state['result'], height = 275)
-
-score = st.text_area("Cosine similarity score", st.session_state['score'], height = 25)
-
-
-
-
-
-# max_length = st.sidebar.slider("Max Length", min_value = 10, max_value=30)
-# temperature = st.sidebar.slider("Temperature", value = 1.0, min_value = 0.0, max_value=1.0, step=0.05)
-# top_k = st.sidebar.slider("Top-k", min_value = 0, max_value=5, value = 0)
-# top_p = st.sidebar.slider("Top-p", min_value = 0.0, max_value=1.0, step = 0.05, value = 0.9)
-# num_return_sequences = st.sidebar.number_input('Number of Return Sequences', min_value=1, max_value=5, value=1, step=1)s
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec776a7d993d37a267ba115d1f7b6ed3f47543372183e0aba99d3ec5cdbf443d
+size 12135

chroma/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c75a8c4781833f9ff6631ce35b3afdf4f14fcfa96dbb89119f82e0ebede75538
+size 94322688

chroma/f7fa132c-535b-432c-a08e-0733244a743d/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a178127ae117d29013c884005a50856af4d162e58e633875f3646a206de6db42
+size 18436000

chroma/f7fa132c-535b-432c-a08e-0733244a743d/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d515f925f64647cf578e98b503441d063c2a4ff18e6cfce00074c7a9a7110c
+size 100

chroma/f7fa132c-535b-432c-a08e-0733244a743d/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46404ed29ee26c1c828c9efe6c3ff2914220029c9fdd37b782725aaf2a648e0
+size 636115

chroma/f7fa132c-535b-432c-a08e-0733244a743d/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74161e53fe79a9fafa54ab0ca4401fa6ec057448b2b7e5e62a71fc3b442bb876
+size 44000

chroma/f7fa132c-535b-432c-a08e-0733244a743d/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd0a9b861ed1b179ba75a1a07bab17f04657fa9102271e900488b0f544c55b0e
+size 97380

requirements.txt

@@ -1,186 +1,3 @@
-accelerate==0.29.2
-aiofiles==23.2.1
-aiohttp==3.9.4
-aiosignal==1.3.1
-altair==5.3.0
-annotated-types==0.6.0
-anyio==4.3.0
-appnope==0.1.4
-asgiref==3.8.1
-asttokens==2.4.1
-async-timeout==4.0.3
-attrs==23.2.0
-backoff==2.2.1
-bcrypt==4.1.2
-bitsandbytes==0.42.0
-blinker==1.7.0
-build==1.2.1
-cachetools==5.3.3
-certifi==2024.2.2
-charset-normalizer==3.3.2
-chroma-hnswlib==0.7.3
-chromadb==0.4.24
-click==8.1.7
-coloredlogs==15.0.1
-comm==0.2.2
-contourpy==1.2.1
-cycler==0.12.1
-dataclasses-json==0.6.4
-debugpy==1.8.1
-decorator==5.1.1
-Deprecated==1.2.14
-distro==1.9.0
-exceptiongroup==1.2.0
-executing==2.0.1
-fastapi==0.110.1
-ffmpy==0.3.2
-filelock==3.13.4
-flatbuffers==24.3.25
-fonttools==4.51.0
-frozenlist==1.4.1
-fsspec==2024.3.1
-gitdb==4.0.11
-GitPython==3.1.43
-google-auth==2.29.0
-googleapis-common-protos==1.63.0
-gradio==4.26.0
-gradio_client==0.15.1
-grpcio==1.62.1
-h11==0.14.0
-httpcore==1.0.5
-httptools==0.6.1
-httpx==0.27.0
-huggingface-hub==0.22.2
-humanfriendly==10.0
-idna==3.7
-importlib-metadata==7.0.0
-importlib_resources==6.4.0
-ipykernel==6.29.4
-ipython==8.18.1
-jedi==0.19.1
-Jinja2==3.1.3
-joblib==1.4.0
-jsonpatch==1.33
-jsonpointer==2.4
-jsonschema==4.21.1
-jsonschema-specifications==2023.12.1
-jupyter_client==8.6.1
-jupyter_core==5.7.2
-kiwisolver==1.4.5
-kubernetes==29.0.0
-langchain==0.1.16
-langchain-chroma==0.1.0
-langchain-community==0.0.33
-langchain-core==0.1.43
-langchain-openai==0.1.3
-langchain-text-splitters==0.0.1
-langsmith==0.1.48
-lark==1.1.9
-markdown-it-py==3.0.0
-MarkupSafe==2.1.5
-marshmallow==3.21.1
-matplotlib==3.8.4
-matplotlib-inline==0.1.7
-mdurl==0.1.2
-mmh3==4.1.0
-monotonic==1.6
-mpmath==1.3.0
-multidict==6.0.5
-mypy-extensions==1.0.0
-nest-asyncio==1.6.0
-networkx==3.2.1
-numpy==1.26.4
-oauthlib==3.2.2
-onnxruntime==1.17.3
-openai==1.19.0
-opentelemetry-api==1.24.0
-opentelemetry-exporter-otlp-proto-common==1.24.0
-opentelemetry-exporter-otlp-proto-grpc==1.24.0
-opentelemetry-instrumentation==0.45b0
-opentelemetry-instrumentation-asgi==0.45b0
-opentelemetry-instrumentation-fastapi==0.45b0
-opentelemetry-proto==1.24.0
-opentelemetry-sdk==1.24.0
-opentelemetry-semantic-conventions==0.45b0
-opentelemetry-util-http==0.45b0
-orjson==3.10.1
-overrides==7.7.0
-packaging==23.2
-pandas==2.2.2
-parso==0.8.4
-pexpect==4.9.0
-pillow==10.3.0
-platformdirs==4.2.0
-posthog==3.5.0
-prompt-toolkit==3.0.43
-protobuf==4.25.3
-psutil==5.9.8
-ptyprocess==0.7.0
-pulsar-client==3.5.0
-pure-eval==0.2.2
-pyarrow==15.0.2
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
-pydantic==2.7.0
-pydantic_core==2.18.1
-pydeck==0.8.1b0
-pydub==0.25.1
-Pygments==2.17.2
-pyparsing==3.1.2
-PyPika==0.48.9
-pyproject_hooks==1.0.0
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
-python-multipart==0.0.9
-pytz==2024.1
-PyYAML==6.0.1
-pyzmq==26.0.0
-referencing==0.34.0
-regex==2023.12.25
-requests==2.31.0
-requests-oauthlib==2.0.0
-rich==13.7.1
-rpds-py==0.18.0
-rsa==4.9
-ruff==0.3.7
-safetensors==0.4.3
-scikit-learn==1.4.2
-scipy==1.13.0
-semantic-version==2.10.0
-sentence-transformers==2.7.0
-shellingham==1.5.4
-six==1.16.0
-smmap==5.0.1
-sniffio==1.3.1
-SQLAlchemy==2.0.29
-stack-data==0.6.3
-starlette==0.37.2
-streamlit==1.33.0
-sympy==1.12
-tenacity==8.2.3
-threadpoolctl==3.4.0
-tiktoken==0.6.0
-tokenizers==0.15.2
-toml==0.10.2
-tomli==2.0.1
-tomlkit==0.12.0
-toolz==0.12.1
-torch==2.2.2
-tornado==6.4
-tqdm==4.66.2
-traitlets==5.14.2
-transformers==4.39.3
-typer==0.12.3
-typing-inspect==0.9.0
-typing_extensions==4.11.0
-tzdata==2024.1
-urllib3==2.2.1
-uvicorn==0.29.0
-uvloop==0.19.0
-watchfiles==0.21.0
-wcwidth==0.2.13
-websocket-client==1.7.0
-websockets==11.0.3
-wrapt==1.16.0
-yarl==1.9.4
-zipp==3.18.1
+version https://git-lfs.github.com/spec/v1
+oid sha256:1df8136f3d57d74ec4a316080a4d6be792614a760dea6473ae9fe3f317c22d01
+size 3490