Upload 18 files

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,81 @@
 ---
-title: Ref
-emoji: 🐢
+title: Fact Checking rocks!
+emoji: 🎸
 colorFrom: purple
-colorTo: purple
+colorTo: blue
 sdk: streamlit
-sdk_version: 1.15.2
-app_file: app.py
-pinned: false
+sdk_version: 1.10.0
+app_file: Rock_fact_checker.py
+pinned: true
+models: [sentence-transformers/msmarco-distilbert-base-tas-b, microsoft/deberta-v2-xlarge-mnli]
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Fact Checking 🎸 Rocks!   [![Generic badge](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue.svg)](https://huggingface.co/spaces/anakin87/fact-checking-rocks) [![Generic badge](https://img.shields.io/github/stars/anakin87/fact-checking-rocks?label=Github&style=social)](https://github.com/anakin87/fact-checking-rocks)
+
+## *Fact checking baseline combining dense retrieval and textual entailment*
+
+- [Fact Checking 🎸 Rocks!    ](#fact-checking--rocks---)
+  - [*Fact checking baseline combining dense retrieval and textual entailment*](#fact-checking-baseline-combining-dense-retrieval-and-textual-entailment)
+    - [Idea](#idea)
+    - [System description](#system-description)
+      - [Indexing pipeline](#indexing-pipeline)
+      - [Search pipeline](#search-pipeline)
+    - [Limits and possible improvements](#limits-and-possible-improvements)
+    - [Repository structure](#repository-structure)
+    - [Installation](#installation)
+
+### Idea
+💡 This project aims to show that a *naive and simple baseline* for fact checking can be built by combining dense retrieval and a textual entailment task.
+In a nutshell, the flow is as follows:
+* the user enters a factual statement
+* the relevant passages are retrieved from the knowledge base using dense retrieval
+* the system computes the text entailment between each relevant passage and the statement, using a Natural Language Inference model
+* the entailment scores are aggregated to produce a summary score.
+
+### System description
+🪄 This project is strongly based on [🔎 Haystack](https://github.com/deepset-ai/haystack), an open source NLP framework to realize search system. The main components of our system are an indexing pipeline and a search pipeline.
+
+
+#### Indexing pipeline
+* [Crawling](https://github.com/anakin87/fact-checking-rocks/blob/321ba7893bbe79582f8c052493acfda497c5b785/notebooks/get_wikipedia_data.ipynb): Crawl data from Wikipedia, starting from the page [List of mainstream rock performers](https://en.wikipedia.org/wiki/List_of_mainstream_rock_performers) and using the [python wrapper](https://github.com/goldsmith/Wikipedia)
+* [Indexing](https://github.com/anakin87/fact-checking-rocks/blob/321ba7893bbe79582f8c052493acfda497c5b785/notebooks/indexing.ipynb)
+  * preprocess the downloaded documents into chunks consisting of 2 sentences
+  * chunks with less than 10 words are discarded, because not very informative
+  * instantiate a [FAISS](https://github.com/facebookresearch/faiss) Document store and store the passages on it
+  * create embeddings for the passages, using a Sentence Transformer model and save them in FAISS. The retrieval task will involve [*asymmetric semantic search*](https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search) (statements to be verified are usually shorter than inherent passages), therefore I choose the model `msmarco-distilbert-base-tas-b`
+  * save FAISS index.
+
+#### Search pipeline
+
+* the user enters a factual statement
+* compute the embedding of the user statement using the same Sentence Transformer used for indexing (`msmarco-distilbert-base-tas-b`)
+* retrieve the K most relevant text passages stored in FAISS (along with their relevance scores)
+* **text entailment task**: compute the text entailment between each text passage (premise) and the user statement (hypotesis), using a Natural Language Inference model (`microsoft/deberta-v2-xlarge-mnli`). For every text passage, we have 3 scores (summing to 1): entailment, contradiction and neutral. *(For this task, I developed a custom Haystack node: `EntailmentChecker`)*
+* aggregate the text entailment scores: compute the weighted average of them, where the weight is the relevance score. **Now it is possible to tell if the knowledge base confirms, is neutral or disproves the user statement.**
+* *empirical consideration: if in the first N passages (N<K),  there is strong evidence of entailment/contradiction (partial aggregate scores > 0.5), it is better not to consider (K-N) less relevant documents.*
+
+### Limits and possible improvements
+ ✨ As mentioned, the current approach to fact checking is simple and naive. Some **structural limits of this approach**:
+  * there is **no statement detection**. In fact, the statement to be verified is chosen by the user. In real-world applications, this step is often necessary.
+  * **Wikipedia is taken as a source of truth**. Unfortunately, Wikipedia does not contain universal knowledge and there is no real guarantee that it is a source of truth. There are certainly very interesting approaches that view a snapshot of the entire web as an uncurated source of knowledge (see [Facebook Research SPHERE](https://arxiv.org/abs/2112.09924)).
+  * Several papers and even our experiments show a general effectiveness of **dense retrieval** in retrieving textual passages for evaluating the user statement. However, there may be cases in which the most useful textual passages for fact checking do not emerge from the simple semantic similarity with the statement to be verified.
+  * **no organic evaluation** was performed, but only manual experiments.
+
+While keeping this simple approach, some **improvements** could be made:
+* For reasons of simplicity and infrastructural limitations, the retrieval uses only a very small portion of the Wikipedia data (artists pages from the [List of mainstream rock performers](https://en.wikipedia.org/wiki/List_of_mainstream_rock_performers)). With these few data available, in many cases the knowledge base remains neutral even with respect to statements about rock albums/songs. Certainly, fact checking **quality could improve by expanding the knowledge base** and possibly extending it to the entire Wikipedia.
+* Both the retriever model and the Natural Language Inference model are general purpose models and have not been fine-tuned for our domain. Undoubtedly they can **show better performance if fine-tuned in the rock music domain**. Particularly, the retriever model might be adapted with low effort, using [Generative Pseudo Labelling](https://haystack.deepset.ai/guides/gpl).
+
+### Repository structure
+* [Rock_fact_checker.py](Rock_fact_checker.py) and [pages folder](./pages/): multi-page Streamlit web app
+* [app_utils folder](./app_utils/): python modules used in the web app
+* [notebooks folder](./notebooks/): Jupyter/Colab notebooks to get Wikipedia data and index the text passages (using Haystack)
+* [data folder](./data/): all necessary data, including original Wikipedia data, FAISS Index and prepared random statements
+
+### Installation
+💻 To install this project locally, follow these steps:
+* `git clone https://github.com/anakin87/fact-checking-rocks`
+* `cd fact-checking-rocks`
+* `pip install -r requirements.txt`
+
+To run the web app, simply type: `streamlit run Rock_fact_checker.py`

Rock_fact_checker.py

@@ -0,0 +1,125 @@
+import random
+import time
+import logging
+from json import JSONDecodeError
+
+import streamlit as st
+
+from app_utils.backend_utils import load_statements, query
+from app_utils.frontend_utils import (
+    set_state_if_absent,
+    reset_results,
+    entailment_html_messages,
+    create_df_for_relevant_snippets,
+    create_ternary_plot,
+    build_sidebar,
+)
+from app_utils.config import RETRIEVER_TOP_K
+
+
+def main():
+    statements = load_statements()
+    build_sidebar()
+
+    # Persistent state
+    set_state_if_absent("statement", "Elvis Presley is alive")
+    set_state_if_absent("answer", "")
+    set_state_if_absent("results", None)
+    set_state_if_absent("raw_json", None)
+    set_state_if_absent("random_statement_requested", False)
+
+    st.write("# Fact Checking 🎸 Rocks!")
+    st.write()
+    st.markdown(
+        """
+    ##### Enter a factual statement about [Rock music](https://en.wikipedia.org/wiki/List_of_mainstream_rock_performers) and let the AI check it out for you...
+    """
+    )
+    # Search bar
+    statement = st.text_input(
+        "", value=st.session_state.statement, max_chars=100, on_change=reset_results
+    )
+    col1, col2 = st.columns(2)
+    col1.markdown(
+        "<style>.stButton button {width:100%;}</style>", unsafe_allow_html=True
+    )
+    col2.markdown(
+        "<style>.stButton button {width:100%;}</style>", unsafe_allow_html=True
+    )
+    # Run button
+    run_pressed = col1.button("Run")
+    # Random statement button
+    if col2.button("Random statement"):
+        reset_results()
+        statement = random.choice(statements)
+        # Avoid picking the same statement twice (the change is not visible on the UI)
+        while statement == st.session_state.statement:
+            statement = random.choice(statements)
+        st.session_state.statement = statement
+        st.session_state.random_statement_requested = True
+        # Re-runs the script setting the random statement as the textbox value
+        # Unfortunately necessary as the Random statement button is _below_ the textbox
+        # Adapted for Streamlit>=1.12.0
+        if hasattr(st, "scriptrunner"):
+            raise st.scriptrunner.script_runner.RerunException(
+                st.scriptrunner.script_requests.RerunData(widget_states=None)
+            )
+        raise st.runtime.scriptrunner.script_runner.RerunException(
+            st.runtime.scriptrunner.script_requests.RerunData(widget_states=None)
+        )
+    else:
+        st.session_state.random_statement_requested = False
+    run_query = (
+        run_pressed or statement != st.session_state.statement
+    ) and not st.session_state.random_statement_requested
+
+    # Get results for query
+    if run_query and statement:
+        time_start = time.time()
+        reset_results()
+        st.session_state.statement = statement
+        with st.spinner("🧠 &nbsp;&nbsp; Performing neural search on documents..."):
+            try:
+                st.session_state.results = query(statement, RETRIEVER_TOP_K)
+                print(f"S: {statement}")
+                time_end = time.time()
+                print(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
+                print(f"elapsed time: {time_end - time_start}")
+            except JSONDecodeError as je:
+                st.error(
+                    "👓 &nbsp;&nbsp; An error occurred reading the results. Is the document store working?"
+                )
+                return
+            except Exception as e:
+                logging.exception(e)
+                st.error("🐞 &nbsp;&nbsp; An error occurred during the request.")
+                return
+
+    # Display results
+    if st.session_state.results:
+        docs = st.session_state.results["documents"]
+        agg_entailment_info = st.session_state.results["aggregate_entailment_info"]
+
+        # show different messages depending on entailment results
+        max_key = max(agg_entailment_info, key=agg_entailment_info.get)
+        message = entailment_html_messages[max_key]
+        st.markdown(f"<br/><h4>{message}</h4>", unsafe_allow_html=True)
+
+        st.markdown(f"###### Aggregate entailment information:")
+        col1, col2 = st.columns([2, 1])
+        fig = create_ternary_plot(agg_entailment_info)
+        with col1:
+            st.plotly_chart(fig, use_container_width=True)
+        with col2:
+            st.write(agg_entailment_info)
+
+        st.markdown(f"###### Most Relevant snippets:")
+        df, urls = create_df_for_relevant_snippets(docs)
+        st.dataframe(df)
+        str_wiki_pages = "Wikipedia source pages: "
+        for doc, url in urls.items():
+            str_wiki_pages += f"[{doc}]({url}) "
+        st.markdown(str_wiki_pages)
+
+
+main()

app_utils/backend_utils.py

@@ -0,0 +1,67 @@
+import shutil
+
+from haystack.document_stores import FAISSDocumentStore
+from haystack.nodes import EmbeddingRetriever
+from haystack.pipelines import Pipeline
+import streamlit as st
+
+from app_utils.entailment_checker import EntailmentChecker
+from app_utils.config import (
+    STATEMENTS_PATH,
+    INDEX_DIR,
+    RETRIEVER_MODEL,
+    RETRIEVER_MODEL_FORMAT,
+    NLI_MODEL,
+)
+
+
+@st.cache()
+def load_statements():
+    """Load statements from file"""
+    with open(STATEMENTS_PATH) as fin:
+        statements = [
+            line.strip() for line in fin.readlines() if not line.startswith("#")
+        ]
+    return statements
+
+
+# cached to make index and models load only at start
+@st.cache(
+    hash_funcs={"builtins.SwigPyObject": lambda _: None}, allow_output_mutation=True
+)
+def start_haystack():
+    """
+    load document store, retriever, entailment checker and create pipeline
+    """
+    shutil.copy(f"{INDEX_DIR}/faiss_document_store.db", ".")
+    document_store = FAISSDocumentStore(
+        faiss_index_path=f"{INDEX_DIR}/my_faiss_index.faiss",
+        faiss_config_path=f"{INDEX_DIR}/my_faiss_index.json",
+    )
+    print(f"Index size: {document_store.get_document_count()}")
+    retriever = EmbeddingRetriever(
+        document_store=document_store,
+        embedding_model=RETRIEVER_MODEL,
+        model_format=RETRIEVER_MODEL_FORMAT,
+    )
+    entailment_checker = EntailmentChecker(
+        model_name_or_path=NLI_MODEL,
+        use_gpu=False,
+        entailment_contradiction_threshold=0.5,
+    )
+
+    pipe = Pipeline()
+    pipe.add_node(component=retriever, name="retriever", inputs=["Query"])
+    pipe.add_node(component=entailment_checker, name="ec", inputs=["retriever"])
+    return pipe
+
+
+pipe = start_haystack()
+
+# the pipeline is not included as parameter of the following function,
+# because it is difficult to cache
+@st.cache(allow_output_mutation=True)
+def query(statement: str, retriever_top_k: int = 5):
+    """Run query and verify statement"""
+    params = {"retriever": {"top_k": retriever_top_k}}
+    return pipe.run(statement, params=params)

app_utils/config.py

@@ -0,0 +1,16 @@
+import streamlit as st
+
+INDEX_DIR = "data/index"
+STATEMENTS_PATH = "data/statements.txt"
+
+RETRIEVER_MODEL = "sentence-transformers/msmarco-distilbert-base-tas-b"
+RETRIEVER_MODEL_FORMAT = "sentence_transformers"
+RETRIEVER_TOP_K = 5
+
+# In HF Space, we use microsoft/deberta-v2-xlarge-mnli
+# for local testing, a smaller model is better
+try:
+    NLI_MODEL = st.secrets["NLI_MODEL"]
+except:
+    NLI_MODEL = "valhalla/distilbart-mnli-12-1"
+print(f"Used NLI model: {NLI_MODEL}")

app_utils/entailment_checker.py

@@ -0,0 +1,109 @@
+from typing import List, Optional
+
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoConfig
+import torch
+from haystack.nodes.base import BaseComponent
+from haystack.modeling.utils import initialize_device_settings
+from haystack.schema import Document
+
+
+class EntailmentChecker(BaseComponent):
+    """
+    This node checks the entailment between every document content and the query.
+    It enrichs the documents metadata with entailment informations.
+    It also returns aggregate entailment information.
+    """
+
+    outgoing_edges = 1
+
+    def __init__(
+        self,
+        model_name_or_path: str = "roberta-large-mnli",
+        model_version: Optional[str] = None,
+        tokenizer: Optional[str] = None,
+        use_gpu: bool = True,
+        batch_size: int = 16,
+        entailment_contradiction_threshold: float = 0.5,
+    ):
+        """
+        Load a Natural Language Inference model from Transformers.
+
+        :param model_name_or_path: Directory of a saved model or the name of a public model.
+        See https://huggingface.co/models for full list of available models.
+        :param model_version: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.
+        :param tokenizer: Name of the tokenizer (usually the same as model)
+        :param use_gpu: Whether to use GPU (if available).
+        :param batch_size: Number of Documents to be processed at a time.
+        :param entailment_contradiction_threshold: if in the first N documents there is a strong evidence of entailment/contradiction
+        (aggregate entailment or contradiction are greater than the threshold), the less relevant documents are not taken into account
+        """
+        super().__init__()
+
+        self.devices, _ = initialize_device_settings(use_cuda=use_gpu, multi_gpu=False)
+
+        tokenizer = tokenizer or model_name_or_path
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer)
+        self.model = AutoModelForSequenceClassification.from_pretrained(
+            pretrained_model_name_or_path=model_name_or_path, revision=model_version
+        )
+        self.batch_size = batch_size
+        self.entailment_contradiction_threshold = entailment_contradiction_threshold
+        self.model.to(str(self.devices[0]))
+
+        id2label = AutoConfig.from_pretrained(model_name_or_path).id2label
+        self.labels = [id2label[k].lower() for k in sorted(id2label)]
+        if "entailment" not in self.labels:
+            raise ValueError(
+                "The model config must contain entailment value in the id2label dict."
+            )
+
+    def run(self, query: str, documents: List[Document]):
+
+        scores, agg_con, agg_neu, agg_ent = 0, 0, 0, 0
+        for i, doc in enumerate(documents):
+            entailment_info = self.get_entailment(premise=doc.content, hypotesis=query)
+            doc.meta["entailment_info"] = entailment_info
+
+            scores += doc.score
+            con, neu, ent = (
+                entailment_info["contradiction"],
+                entailment_info["neutral"],
+                entailment_info["entailment"],
+            )
+            agg_con += con * doc.score
+            agg_neu += neu * doc.score
+            agg_ent += ent * doc.score
+
+            # if in the first documents there is a strong evidence of entailment/contradiction,
+            # there is no need to consider less relevant documents
+            if max(agg_con, agg_ent) / scores > self.entailment_contradiction_threshold:
+                break
+
+        aggregate_entailment_info = {
+            "contradiction": round(agg_con / scores, 2),
+            "neutral": round(agg_neu / scores, 2),
+            "entailment": round(agg_ent / scores, 2),
+        }
+
+        entailment_checker_result = {
+            "documents": documents[: i + 1],
+            "aggregate_entailment_info": aggregate_entailment_info,
+        }
+
+        return entailment_checker_result, "output_1"
+
+    def run_batch(self, queries: List[str], documents: List[Document]):
+        pass
+
+    def get_entailment(self, premise, hypotesis):
+        with torch.inference_mode():
+            inputs = self.tokenizer(
+                f"{premise}{self.tokenizer.sep_token}{hypotesis}", return_tensors="pt"
+            ).to(self.devices[0])
+            out = self.model(**inputs)
+            logits = out.logits
+            probs = (
+                torch.nn.functional.softmax(logits, dim=-1)[0, :].detach().cpu().numpy()
+            )
+        entailment_dict = {k.lower(): v for k, v in zip(self.labels, probs)}
+        return entailment_dict

app_utils/frontend_utils.py

@@ -0,0 +1,120 @@
+import streamlit as st
+import pandas as pd
+import plotly.graph_objects as go
+
+
+entailment_html_messages = {
+    "entailment": 'The knowledge base seems to <span style="color:green">confirm</span> your statement',
+    "contradiction": 'The knowledge base seems to <span style="color:red">contradict</span> your statement',
+    "neutral": 'The knowledge base is <span style="color:darkgray">neutral</span> about your statement',
+}
+
+
+def build_sidebar():
+    sidebar = """
+    <h1 style='text-align: center'>Fact Checking 🎸 Rocks!</h1>
+    <div style='text-align: center'>
+    <i>Fact checking baseline combining dense retrieval and textual entailment</i>
+    <p><br/><a href='https://github.com/anakin87/fact-checking-rocks'>Github project</a> - Based on <a href='https://github.com/deepset-ai/haystack'>Haystack</a></p>
+    <p><small><a href='https://en.wikipedia.org/wiki/List_of_mainstream_rock_performers'>Data crawled from Wikipedia</a></small></p>
+    </div>
+    """
+    st.sidebar.markdown(sidebar, unsafe_allow_html=True)
+
+
+def set_state_if_absent(key, value):
+    if key not in st.session_state:
+        st.session_state[key] = value
+
+
+# Small callback to reset the interface in case the text of the question changes
+def reset_results(*args):
+    st.session_state.answer = None
+    st.session_state.results = None
+    st.session_state.raw_json = None
+
+
+def create_ternary_plot(entailment_data):
+    """
+    Create a Plotly ternary plot for the given entailment dict.
+    """
+    hover_text = ""
+    for label, value in entailment_data.items():
+        hover_text += f"{label}: {value}<br>"
+
+    fig = go.Figure(
+        go.Scatterternary(
+            {
+                "cliponaxis": False,
+                "mode": "markers",
+                "a": [i for i in map(lambda x: x["entailment"], [entailment_data])],
+                "b": [i for i in map(lambda x: x["contradiction"], [entailment_data])],
+                "c": [i for i in map(lambda x: x["neutral"], [entailment_data])],
+                "hoverinfo": "text",
+                "text": hover_text,
+                "marker": {
+                    "color": "#636efa",
+                    "size": [0.01],
+                    "sizemode": "area",
+                    "sizeref": 2.5e-05,
+                    "symbol": "circle",
+                },
+            }
+        )
+    )
+
+    fig.update_layout(
+        {
+            "ternary": {
+                "sum": 1,
+                "aaxis": makeAxis("Entailment", 0),
+                "baxis": makeAxis("<br>Contradiction", 45),
+                "caxis": makeAxis("<br>Neutral", -45),
+            }
+        }
+    )
+    return fig
+
+
+def makeAxis(title, tickangle):
+    return {
+        "title": title,
+        "titlefont": {"size": 20},
+        "tickangle": tickangle,
+        "tickcolor": "rgba(0,0,0,0)",
+        "ticklen": 5,
+        "showline": False,
+        "showgrid": True,
+    }
+
+
+def create_df_for_relevant_snippets(docs):
+    """
+    Create a dataframe that contains all relevant snippets.
+    Also returns the URLs
+    """
+    rows = []
+    urls = {}
+    for doc in docs:
+        row = {
+            "Title": doc.meta["name"],
+            "Relevance": f"{doc.score:.3f}",
+            "con": f"{doc.meta['entailment_info']['contradiction']:.2f}",
+            "neu": f"{doc.meta['entailment_info']['neutral']:.2f}",
+            "ent": f"{doc.meta['entailment_info']['entailment']:.2f}",
+            "Content": doc.content,
+        }
+        urls[doc.meta["name"]] = doc.meta["url"]
+        rows.append(row)
+        df = pd.DataFrame(rows)
+        df["Content"] = df["Content"].str.wrap(75)
+        df = df.style.apply(highlight_cols)
+
+    return df, urls
+
+
+def highlight_cols(s):
+    coldict = {"con": "#FFA07A", "neu": "#E5E4E2", "ent": "#a9d39e"}
+    if s.name in coldict.keys():
+        return ["background-color: {}".format(coldict[s.name])] * len(s)
+    return [""] * len(s)

data/index/my_faiss_index.json

@@ -0,0 +1 @@
+{"similarity": "dot_product", "embedding_dim": 768}

data/statements.txt

@@ -0,0 +1 @@
+Bring your friend and you will get referral bonus

notebooks/get_wikipedia_data.ipynb

@@ -0,0 +1,582 @@
+{
+ "metadata": {
+  "kernelspec": {
+   "language": "python",
+   "display_name": "Python 3",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.7.12",
+   "mimetype": "text/x-python",
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "pygments_lexer": "ipython3",
+   "nbconvert_exporter": "python",
+   "file_extension": ".py"
+  }
+ },
+ "nbformat_minor": 4,
+ "nbformat": 4,
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": "# Download data from Wikipedia",
+   "metadata": {}
+  },
+  {
+   "cell_type": "code",
+   "source": "# install wikipedia API python wrapper\n! pip install wikipedia",
+   "metadata": {
+    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
+    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
+    "execution": {
+     "iopub.status.busy": "2022-08-20T21:43:59.293655Z",
+     "iopub.execute_input": "2022-08-20T21:43:59.294792Z",
+     "iopub.status.idle": "2022-08-20T21:44:15.263363Z",
+     "shell.execute_reply.started": "2022-08-20T21:43:59.294746Z",
+     "shell.execute_reply": "2022-08-20T21:44:15.262171Z"
+    },
+    "trusted": true
+   },
+   "execution_count": 3,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": "import wikipedia\nimport json\nimport traceback",
+   "metadata": {
+    "execution": {
+     "iopub.status.busy": "2022-08-20T21:44:15.265341Z",
+     "iopub.execute_input": "2022-08-20T21:44:15.265753Z",
+     "iopub.status.idle": "2022-08-20T21:44:15.470330Z",
+     "shell.execute_reply.started": "2022-08-20T21:44:15.265709Z",
+     "shell.execute_reply": "2022-08-20T21:44:15.468665Z"
+    },
+    "trusted": true
+   },
+   "execution_count": 4,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "# titles to download, from https://en.wikipedia.org/wiki/List_of_mainstream_rock_performers\n",
+    "\n",
+    "pages_titles = \"\"\"10cc\n",
+    "10_Years_(band)\n",
+    "3_Doors_Down\n",
+    "311_(band)\n",
+    "38_Special_(band)\n",
+    "Accept_(band)\n",
+    "AC/DC\n",
+    "Bryan_Adams\n",
+    "Aerosmith\n",
+    "AFI_(band)\n",
+    "Air_Supply\n",
+    "The_Alan_Parsons_Project\n",
+    "Alice_in_Chains\n",
+    "The_All-American_Rejects\n",
+    "The_Allman_Brothers_Band\n",
+    "Alter_Bridge\n",
+    "Ambrosia_(band)\n",
+    "America_(band)\n",
+    "The_Animals\n",
+    "Adam_Ant\n",
+    "Anthrax_(American_band)\n",
+    "April_Wine\n",
+    "Arcade_Fire\n",
+    "Arctic_Monkeys\n",
+    "Asia_(band)\n",
+    "Audioslave\n",
+    "Avenged_Sevenfold\n",
+    "Awolnation\n",
+    "The_B-52's\n",
+    "Bachman–Turner_Overdrive\n",
+    "Bad_Company\n",
+    "Badfinger\n",
+    "The_Band\n",
+    "The_Bangles\n",
+    "Barenaked_Ladies\n",
+    "Bay_City_Rollers\n",
+    "The_Beach_Boys\n",
+    "The_Beatles\n",
+    "Beck\n",
+    "Ben_Folds_Five\n",
+    "Pat_Benatar\n",
+    "Chuck_Berry\n",
+    "The_Big_Bopper\n",
+    "Billy_Talent\n",
+    "The_Black_Crowes\n",
+    "The_Black_Keys\n",
+    "Black_Sabbath\n",
+    "Black_Stone_Cherry\n",
+    "Black_Veil_Brides\n",
+    "Blink-182\n",
+    "Bloodhound_Gang\n",
+    "Blue_October\n",
+    "Blue_Öyster_Cult\n",
+    "Blues_Traveler\n",
+    "James_Blunt\n",
+    "Blur_(band)\n",
+    "Bon_Jovi\n",
+    "Boston_(band)\n",
+    "David_Bowie\n",
+    "Bowling_for_Soup\n",
+    "Boys_Like_Girls\n",
+    "Bread_(band)\n",
+    "Breaking_Benjamin\n",
+    "Bring_Me_the_Horizon\n",
+    "Jackson_Browne\n",
+    "Buckcherry\n",
+    "Jeff_Buckley\n",
+    "Bullet_for_My_Valentine\n",
+    "Bush_(British_band)\n",
+    "The_Byrds\n",
+    "Cage_the_Elephant\n",
+    "Cake_(band)\n",
+    "Canned_Heat\n",
+    "The_Cab\n",
+    "The_Cardigans\n",
+    "The_Cars\n",
+    "Catfish_and_the_Bottlemen\n",
+    "Harry_Chapin\n",
+    "Tracy_Chapman\n",
+    "Cheap_Trick\n",
+    "Chevelle_(band)\n",
+    "Chicago_(band)\n",
+    "Chubby_Checker\n",
+    "Cinderella_(band)\n",
+    "Dallas_Green_(musician)\n",
+    "Eric_Clapton\n",
+    "The_Clash\n",
+    "Eddie_Cochran\n",
+    "Joe_Cocker\n",
+    "Coheed_and_Cambria\n",
+    "Cold_Chisel\n",
+    "Coldplay\n",
+    "Collective_Soul\n",
+    "Phil_Collins\n",
+    "Alice_Cooper\n",
+    "Chris_Cornell\n",
+    "Elvis_Costello\n",
+    "Counting_Crows\n",
+    "The_Cranberries\n",
+    "Crash_Test_Dummies\n",
+    "Cream_(band)\n",
+    "Creed_(band)\n",
+    "Creedence_Clearwater_Revival\n",
+    "Jim_Croce\n",
+    "Crosby,_Stills,_Nash_&_Young\n",
+    "Christopher_Cross\n",
+    "Sheryl_Crow\n",
+    "Crowded_House\n",
+    "The_Cult\n",
+    "The_Cure\n",
+    "Damn_Yankees_(band)\n",
+    "Dashboard_Confessional\n",
+    "Daughtry_(band)\n",
+    "The_Dave_Clark_Five\n",
+    "Dave_Matthews_Band\n",
+    "Days_of_the_New\n",
+    "Death_Cab_for_Cutie\n",
+    "Deep_Purple\n",
+    "Def_Leppard\n",
+    "Deftones\n",
+    "Depeche_Mode\n",
+    "Bo_Diddley\n",
+    "Dio_(band)\n",
+    "Dire_Straits\n",
+    "Disturbed_(band)\n",
+    "Fats_Domino\n",
+    "Donovan\n",
+    "The_Doobie_Brothers\n",
+    "The_Doors\n",
+    "Dr._Hook_&_the_Medicine_Show\n",
+    "Dropkick_Murphys\n",
+    "Drowning_Pool\n",
+    "Duran_Duran\n",
+    "Ian_Dury\n",
+    "Bob_Dylan\n",
+    "Eagles_(band)\n",
+    "Echo_&_the_Bunnymen\n",
+    "Duane_Eddy\n",
+    "Edgar_Winter\n",
+    "Electric_Light_Orchestra\n",
+    "Emerson,_Lake_&_Palmer\n",
+    "England_Dan_&_John_Ford_Coley\n",
+    "Melissa_Etheridge\n",
+    "Europe_(band)\n",
+    "Evanescence\n",
+    "Everclear_(band)\n",
+    "Everlast\n",
+    "The_Everly_Brothers\n",
+    "Extreme_(band)\n",
+    "Faces_(band)\n",
+    "Faith_No_More\n",
+    "Fall_Out_Boy\n",
+    "Bryan_Ferry\n",
+    "Filter_(band)\n",
+    "Finger_Eleven\n",
+    "FireHouse\n",
+    "Five_Finger_Death_Punch\n",
+    "Five_for_Fighting\n",
+    "The_Fixx\n",
+    "The_Flaming_Lips\n",
+    "Fleetwood_Mac\n",
+    "Flogging_Molly\n",
+    "Florence_and_the_Machine\n",
+    "Flyleaf_(band)\n",
+    "Foals_(band)\n",
+    "Dan_Fogelberg\n",
+    "John_Fogerty\n",
+    "Foo_Fighters\n",
+    "Foreigner_(band)\n",
+    "Foster_the_People\n",
+    "The_Four_Seasons_(band)\n",
+    "Peter_Frampton\n",
+    "Franz_Ferdinand_(band)\n",
+    "The_Fray\n",
+    "Glenn_Frey\n",
+    "Fuel_(band)\n",
+    "Fun_(band)\n",
+    "Peter_Gabriel\n",
+    "Garbage_(band)\n",
+    "Genesis_(band)\n",
+    "Ghost_(Swedish_band)\n",
+    "Gin_Blossoms\n",
+    "Gary_Glitter\n",
+    "The_Go-Go's\n",
+    "Godsmack\n",
+    "Golden_Earring\n",
+    "Goo_Goo_Dolls\n",
+    "Good_Charlotte\n",
+    "Grand_Funk_Railroad\n",
+    "Grateful_Dead\n",
+    "Great_White\n",
+    "Green_Day\n",
+    "Greta_Van_Fleet\n",
+    "The_Guess_Who\n",
+    "Guns_N'_Roses\n",
+    "Halestorm\n",
+    "Bill_Haley_&_His_Comets\n",
+    "Hall_&_Oates\n",
+    "George_Harrison\n",
+    "Heart_(band)\n",
+    "Jimi_Hendrix\n",
+    "Don_Henley\n",
+    "Herman's_Hermits\n",
+    "Highly_Suspect\n",
+    "Hinder\n",
+    "The_Hives\n",
+    "Hole_(band)\n",
+    "The_Hollies\n",
+    "Buddy_Holly\n",
+    "Hoobastank\n",
+    "Hootie_&_the_Blowfish\n",
+    "Icehouse_(band)\n",
+    "Billy_Idol\n",
+    "Imagine_Dragons\n",
+    "Incubus_(band)\n",
+    "Interpol_(band)\n",
+    "INXS\n",
+    "Iron_Maiden\n",
+    "The_J._Geils_Band\n",
+    "The_Jam\n",
+    "Tommy_James_and_the_Shondells\n",
+    "Jane's_Addiction\n",
+    "Jefferson_Airplane\n",
+    "Jefferson_Starship\n",
+    "The_Jesus_and_Mary_Chain\n",
+    "Jet_(Australian_band)\n",
+    "Jethro_Tull_(band)\n",
+    "Joan_Jett\n",
+    "Jimmy_Eat_World\n",
+    "Billy_Joel\n",
+    "Elton_John\n",
+    "Janis_Joplin\n",
+    "Journey_(band)\n",
+    "Joy_Division\n",
+    "Judas_Priest\n",
+    "Kaiser_Chiefs\n",
+    "Kaleo_(band)\n",
+    "Kansas_(band)\n",
+    "Keane_(band)\n",
+    "Kid_Rock\n",
+    "The_Killers\n",
+    "Killswitch_Engage\n",
+    "Kings_of_Leon\n",
+    "The_Kinks\n",
+    "Kiss_(band)\n",
+    "Korn\n",
+    "Lenny_Kravitz\n",
+    "Lacuna_Coil\n",
+    "Lamb_of_God_(band)\n",
+    "Avril_Lavigne\n",
+    "Led_Zeppelin\n",
+    "John_Lennon\n",
+    "Huey_Lewis_and_the_News\n",
+    "Jerry_Lee_Lewis\n",
+    "Lifehouse_(band)\n",
+    "Limp_Bizkit\n",
+    "Linkin_Park\n",
+    "Little_Richard\n",
+    "Little_River_Band\n",
+    "Live_(band)\n",
+    "Living_Colour\n",
+    "Kenny_Loggins\n",
+    "Loverboy\n",
+    "The_Lovin'_Spoonful\n",
+    "The_Lumineers\n",
+    "Lynyrd_Skynyrd\n",
+    "The_Mamas_&_the_Papas\n",
+    "Marilyn_Manson\n",
+    "The_Marshall_Tucker_Band\n",
+    "Matchbox_Twenty\n",
+    "John_Mayer\n",
+    "Paul_McCartney\n",
+    "Meat_Loaf\n",
+    "Megadeth\n",
+    "John_Mellencamp\n",
+    "Men_at_Work\n",
+    "Metallica\n",
+    "Midnight_Oil\n",
+    "Mike_and_the_Mechanics\n",
+    "Modest_Mouse\n",
+    "Eddie_Money\n",
+    "The_Monkees\n",
+    "The_Moody_Blues\n",
+    "Alanis_Morissette\n",
+    "Van_Morrison\n",
+    "Morrissey\n",
+    "Mötley_Crüe\n",
+    "Motörhead\n",
+    "Mudvayne\n",
+    "Mumford_&_Sons\n",
+    "Muse_(band)\n",
+    "My_Chemical_Romance\n",
+    "Nickelback\n",
+    "Stevie_Nicks\n",
+    "Harry_Nilsson\n",
+    "Nine_Inch_Nails\n",
+    "Nirvana_(band)\n",
+    "No_Doubt\n",
+    "Ted_Nugent\n",
+    "Oasis_(band)\n",
+    "The_Offspring\n",
+    "Roy_Orbison\n",
+    "Ozzy_Osbourne\n",
+    "Our_Lady_Peace\n",
+    "The_Outfield\n",
+    "P.O.D.\n",
+    "Panic!_at_the_Disco\n",
+    "Pantera\n",
+    "Papa_Roach\n",
+    "Paramore\n",
+    "Pearl_Jam\n",
+    "A_Perfect_Circle\n",
+    "Tom_Petty_and_the_Heartbreakers\n",
+    "Pink_Floyd\n",
+    "Pixies_(band)\n",
+    "Robert_Plant\n",
+    "Poison_(American_band)\n",
+    "The_Police\n",
+    "Iggy_Pop\n",
+    "Pop_Evil\n",
+    "The_Presidents_of_the_United_States_of_America_(band)\n",
+    "The_Pretenders\n",
+    "Elvis_Presley\n",
+    "The_Pretty_Reckless\n",
+    "Primus_(band)\n",
+    "Puddle_of_Mudd\n",
+    "Queen_(band)\n",
+    "Queens_of_the_Stone_Age\n",
+    "Queensrÿche\n",
+    "Quiet_Riot\n",
+    "R.E.M.\n",
+    "Radiohead\n",
+    "Rage_Against_the_Machine\n",
+    "Rainbow_(rock_band)\n",
+    "Rammstein\n",
+    "Ramones\n",
+    "Red_Hot_Chili_Peppers\n",
+    "Lou_Reed\n",
+    "REO_Speedwagon\n",
+    "Rise_Against\n",
+    "The_Rolling_Stones\n",
+    "Linda_Ronstadt\n",
+    "Roxy_Music\n",
+    "Royal_Blood_(band)\n",
+    "Rush_(band)\n",
+    "Saliva_(band)\n",
+    "Sam_Fender\n",
+    "Santana_(band)\n",
+    "Joe_Satriani\n",
+    "Saving_Abel\n",
+    "Scorpions_(band)\n",
+    "The_Script\n",
+    "Seether\n",
+    "Bob_Seger\n",
+    "Sepultura\n",
+    "Sex_Pistols\n",
+    "Shakin'_Stevens\n",
+    "Shinedown\n",
+    "Silverchair\n",
+    "Simon_&_Garfunkel\n",
+    "Simple_Minds\n",
+    "Simple_Plan\n",
+    "Skid_Row_(American_band)\n",
+    "Skillet_(band)\n",
+    "Slade\n",
+    "Slayer\n",
+    "Slipknot_(band)\n",
+    "Small_Faces\n",
+    "Smash_Mouth\n",
+    "The_Smashing_Pumpkins\n",
+    "The_Smiths\n",
+    "Smokie_(band)\n",
+    "Snow_Patrol\n",
+    "Social_Distortion\n",
+    "Soundgarden\n",
+    "Bruce_Springsteen\n",
+    "Billy_Squier\n",
+    "Staind\n",
+    "Ringo_Starr\n",
+    "Starset\n",
+    "Starship_(band)\n",
+    "Status_Quo_(band)\n",
+    "Steely_Dan\n",
+    "Steppenwolf_(band)\n",
+    "Steve_Miller_Band\n",
+    "Rod_Stewart\n",
+    "Sting_(musician)\n",
+    "The_Stone_Roses\n",
+    "Stone_Sour\n",
+    "Stone_Temple_Pilots\n",
+    "The_Strokes\n",
+    "Styx_(band)\n",
+    "Sublime_(band)\n",
+    "Sum_41\n",
+    "Supertramp\n",
+    "Survivor_(band)\n",
+    "The_Sweet\n",
+    "System_of_a_Down\n",
+    "T._Rex_(band)\n",
+    "Talking_Heads\n",
+    "James_Taylor\n",
+    "Tenacious_D\n",
+    "Tesla_(band)\n",
+    "Theory_of_a_Deadman\n",
+    "Thin_Lizzy\n",
+    "Third_Eye_Blind\n",
+    "Thirty_Seconds_to_Mars\n",
+    "George_Thorogood\n",
+    "Thousand_Foot_Krutch\n",
+    "Three_Days_Grace\n",
+    "Three_Dog_Night\n",
+    "Tool_(band)\n",
+    "Toto_(band)\n",
+    "Traffic_(band)\n",
+    "The_Tragically_Hip\n",
+    "Train_(band)\n",
+    "Traveling_Wilburys\n",
+    "Travis_(band)\n",
+    "Trivium_(band)\n",
+    "Twenty_One_Pilots\n",
+    "Twisted_Sister\n",
+    "U2\n",
+    "Uriah_Heep_(band)\n",
+    "The_Used\n",
+    "Steve_Vai\n",
+    "Ritchie_Valens\n",
+    "Vampire_Weekend\n",
+    "Van_Halen\n",
+    "Stevie_Ray_Vaughan\n",
+    "Velvet_Revolver\n",
+    "The_Velvet_Underground\n",
+    "The_Verve\n",
+    "Volbeat\n",
+    "Joe_Walsh\n",
+    "Warrant_(American_band)\n",
+    "Weezer\n",
+    "Jack_White\n",
+    "The_White_Stripes\n",
+    "White_Zombie_(band)\n",
+    "Whitesnake\n",
+    "The_Who\n",
+    "Paul_McCartney_and_Wings\n",
+    "Steve_Winwood\n",
+    "The_Yardbirds\n",
+    "Yes_(band)\n",
+    "Neil_Young\n",
+    "Frank_Zappa\n",
+    "Rob_Zombie\n",
+    "The_Zombies\n",
+    "ZZ_Top\"\"\".split(\n",
+    "    \"\\n\"\n",
+    ")"
+   ],
+   "metadata": {
+    "execution": {
+     "iopub.status.busy": "2022-08-20T23:34:24.681697Z",
+     "iopub.execute_input": "2022-08-20T23:34:24.682223Z",
+     "iopub.status.idle": "2022-08-20T23:34:24.693942Z",
+     "shell.execute_reply.started": "2022-08-20T23:34:24.682178Z",
+     "shell.execute_reply": "2022-08-20T23:34:24.693004Z"
+    },
+    "trusted": true
+   },
+   "execution_count": 54,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "for i, raw_title in enumerate(pages_titles):\n",
+    "    if i % 10 == 0:\n",
+    "        print(i / len(pages_titles) * 100)\n",
+    "    try:\n",
+    "        page = wikipedia.page(title=raw_title.replace(\"_\", \" \"), auto_suggest=False)\n",
+    "        id_ = page.pageid\n",
+    "        url = page.url\n",
+    "        dic = {\"content\": page.content, \"meta\": {\"name\": page.title, \"url\": url}}\n",
+    "\n",
+    "        with open(f\"/kaggle/working/rock_wiki/{id_}.json\", \"w\") as fo:\n",
+    "            json.dump(dic, fo)\n",
+    "    except Exception as e:\n",
+    "        traceback.print_exc()\n",
+    "        print(raw_title)"
+   ],
+   "metadata": {
+    "execution": {
+     "iopub.status.busy": "2022-08-20T23:34:49.157641Z",
+     "iopub.execute_input": "2022-08-20T23:34:49.158086Z",
+     "iopub.status.idle": "2022-08-20T23:44:29.346317Z",
+     "shell.execute_reply.started": "2022-08-20T23:34:49.158047Z",
+     "shell.execute_reply": "2022-08-20T23:44:29.345032Z"
+    },
+    "trusted": true
+   },
+   "execution_count": 57,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "source": "! tar -czvf rock_wiki.tar.gz ./rock_wiki",
+   "metadata": {
+    "execution": {
+     "iopub.status.busy": "2022-08-20T23:50:44.643851Z",
+     "iopub.execute_input": "2022-08-20T23:50:44.644378Z",
+     "iopub.status.idle": "2022-08-20T23:50:44.650366Z",
+     "shell.execute_reply.started": "2022-08-20T23:50:44.644328Z",
+     "shell.execute_reply": "2022-08-20T23:50:44.649169Z"
+    },
+    "trusted": true
+   },
+   "execution_count": 60,
+   "outputs": []
+  }
+ ]
+}

notebooks/indexing.ipynb

@@ -0,0 +1,417 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Indexing\n",
+    "Using [Haystack](https://github.com/deepset-ai/haystack), the following steps are performed:\n",
+    "- load and preprocess documents downloaded from Wikipedia\n",
+    "- create document store and write documents\n",
+    "- initialize retriever and generate document embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
+    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "! pip install farm-haystack[faiss-gpu]==1.7.0"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:23.692554Z",
+     "iopub.status.busy": "2022-08-21T08:23:23.692208Z",
+     "iopub.status.idle": "2022-08-21T08:23:23.700721Z",
+     "shell.execute_reply": "2022-08-21T08:23:23.698130Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:23.692512Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "import glob, json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:23.707774Z",
+     "iopub.status.busy": "2022-08-21T08:23:23.704107Z",
+     "iopub.status.idle": "2022-08-21T08:23:25.026910Z",
+     "shell.execute_reply": "2022-08-21T08:23:25.025990Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:23.705010Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "docs = []\n",
+    "\n",
+    "for json_file in glob.glob(\"../input/crawl-rock/rock_wiki/*.json\"):\n",
+    "    with open(json_file, \"r\") as fin:\n",
+    "        doc = json.load(fin)\n",
+    "\n",
+    "    docs.append(doc)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:25.030530Z",
+     "iopub.status.busy": "2022-08-21T08:23:25.029931Z",
+     "iopub.status.idle": "2022-08-21T08:23:25.039324Z",
+     "shell.execute_reply": "2022-08-21T08:23:25.037960Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:25.030491Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "453"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Preprocess documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:25.050479Z",
+     "iopub.status.busy": "2022-08-21T08:23:25.050099Z",
+     "iopub.status.idle": "2022-08-21T08:23:42.089083Z",
+     "shell.execute_reply": "2022-08-21T08:23:42.087929Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:25.050446Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "108e8c46426f44e7be98a8ae930d81ce",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Preprocessing:   0%|          | 0/453 [00:00<?, ?docs/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# preprocess documents, splitting by chunks of 2 sentences\n",
+    "\n",
+    "from haystack.nodes import PreProcessor\n",
+    "\n",
+    "processor = PreProcessor(\n",
+    "    clean_empty_lines=True,\n",
+    "    clean_whitespace=True,\n",
+    "    clean_header_footer=True,\n",
+    "    split_by=\"sentence\",\n",
+    "    split_length=2,\n",
+    "    split_respect_sentence_boundary=False,\n",
+    "    split_overlap=0,\n",
+    "    language=\"en\",\n",
+    ")\n",
+    "preprocessed_docs = processor.process(docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:42.092031Z",
+     "iopub.status.busy": "2022-08-21T08:23:42.090654Z",
+     "iopub.status.idle": "2022-08-21T08:23:42.105757Z",
+     "shell.execute_reply": "2022-08-21T08:23:42.104500Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:42.091989Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "50024"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(preprocessed_docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:42.108367Z",
+     "iopub.status.busy": "2022-08-21T08:23:42.107604Z",
+     "iopub.status.idle": "2022-08-21T08:23:42.117080Z",
+     "shell.execute_reply": "2022-08-21T08:23:42.115996Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:42.108271Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[<Document: {'content': 'Disturbed is an American heavy metal band from Chicago, formed in 1994. The band includes vocalist David Draiman, guitarist/keyboardist Dan Donegan, bassist John Moyer, and drummer Mike Wengren.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 0}, 'embedding': None, 'id': '543d4f9f9023bfc277edf307a6aef870'}>,\n",
+       " <Document: {'content': 'Donegan and Wengren have been involved in the band since its inception, with Moyer replacing former bassist Steve \"Fuzz\" Kmak and Draiman replacing original lead vocalist Erich Awalt. The band has released seven studio albums, five of which have consecutively debuted at number one on the Billboard 200.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 1}, 'embedding': None, 'id': 'dfb0ef877837c95b2e8b03cfe2ae2057'}>,\n",
+       " <Document: {'content': \"Disturbed went into hiatus in October 2011, during which the band's members focused on various side projects, and returned in June 2015, releasing their first album in four years, Immortalized in August 2015. They also released two live albums, Music as a Weapon II in February 2004 and Disturbed: Live at Red Rocks in November 2016.\", 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 2}, 'embedding': None, 'id': 'e498da0cc7477f698f4a30c85dbfd95d'}>,\n",
+       " <Document: {'content': 'With over 17 million records sold worldwide, Disturbed ranks alongside Slipknot and Godsmack as one of the most successful rock bands of the 21st century. == History ==\\n\\n=== Early years (1994–1996) ===\\nBefore David Draiman joined Disturbed, guitarist Dan Donegan, drummer Mike Wengren and bassist Steve \"Fuzz\" Kmak were in a band called Brawl with vocalist Erich Awalt.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 3}, 'embedding': None, 'id': '2b51b8f38befc2c53c65c66af6de1e05'}>,\n",
+       " <Document: {'content': 'Before changing their name to \"Brawl\", however, Donegan mentioned in the band\\'s DVD, Decade of Disturbed, that the name was originally going to be \"Crawl\"; they switched it to \"Brawl\", due to the name already being used by another band. Awalt left the band shortly after the recording of a demo tape; the other three members advertised for a singer.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 4}, 'embedding': None, 'id': 'c9b3a4a74f8332c9d4c3d1e30cac49f7'}>,\n",
+       " <Document: {'content': 'They posted an advertisement in the local music publication in Chicago, Illinois, called the \"Illinois Entertainer\". Draiman answered the advertisement after going to twenty other auditions that month.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 5}, 'embedding': None, 'id': '946a4a27f2f1838ec070a951dab2e1b0'}>,\n",
+       " <Document: {'content': 'Guitarist Dan Donegan commented on Draiman: \"You know, out of all the singers that we had talked to or auditioned, he [Draiman] was the only singer who was ready to go with originals. And that impressed me, just to attempt that\".With regard to Draiman being the singer for the band, Donegan said, \"After a minute or two, he just starts banging out these melodies that were huge...I\\'m playing my guitar and I\\'m grinning from ear to ear, trying not to give it away that I like this guy, you know, because I don\\'t want to, you know...[say] \\'Yeah, we\\'ll give you a call back.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 6}, 'embedding': None, 'id': 'ac2bea954f4d19cc0e48868fb503e23e'}>,\n",
+       " <Document: {'content': \"We'll, you know, discuss it.' But I was so psyched.\", 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 7}, 'embedding': None, 'id': '852a32a0d3ce1eefa48d6420fab35dba'}>,\n",
+       " <Document: {'content': 'Chill up my spine. I\\'m like, \\'There is something here.\\'\"', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 8}, 'embedding': None, 'id': 'ebb584a0fee18b51cd14ceadcb4a7bb8'}>,\n",
+       " <Document: {'content': 'As drummer Mike Wengren commented, \"We clicked right off the bat.\" Draiman then joined the band in 1996 and the band was renamed Disturbed.', 'content_type': 'text', 'score': None, 'meta': {'name': 'Disturbed (band)', 'url': 'https://en.wikipedia.org/wiki/Disturbed_(band)', '_split_id': 9}, 'embedding': None, 'id': '51340b7bf229e5b8d4460341f7aaa9d0'}>]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "preprocessed_docs[:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# select only documents with at least 10 words. Otherwise, the documents are not very informative\n",
+    "preprocessed_docs = [doc for doc in preprocessed_docs if len(doc.content.split()) >= 10]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create document store ([FAISS](https://github.com/facebookresearch/faiss)) and write documents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:42.119585Z",
+     "iopub.status.busy": "2022-08-21T08:23:42.118544Z",
+     "iopub.status.idle": "2022-08-21T08:23:42.124669Z",
+     "shell.execute_reply": "2022-08-21T08:23:42.123597Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:42.119551Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "from haystack.document_stores import FAISSDocumentStore\n",
+    "from haystack.nodes import EmbeddingRetriever"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:23:42.129562Z",
+     "iopub.status.busy": "2022-08-21T08:23:42.128772Z",
+     "iopub.status.idle": "2022-08-21T08:23:42.259879Z",
+     "shell.execute_reply": "2022-08-21T08:23:42.258950Z",
+     "shell.execute_reply.started": "2022-08-21T08:23:42.129518Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "# the document store settings are those compatible with Embedding Retriever\n",
+    "document_store = FAISSDocumentStore(similarity=\"dot_product\", embedding_dim=768)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:43:25.952230Z",
+     "iopub.status.busy": "2022-08-21T08:43:25.951856Z",
+     "iopub.status.idle": "2022-08-21T08:46:12.506842Z",
+     "shell.execute_reply": "2022-08-21T08:46:12.505845Z",
+     "shell.execute_reply.started": "2022-08-21T08:43:25.952198Z"
+    },
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dbd72ecf0d36401ba26826f7d9a42540",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Writing Documents:   0%|          | 0/50024 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# write documents\n",
+    "document_store.write_documents(preprocessed_docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Initialize retriever (Embedding Retriever) and generate document embeddings\n",
+    "We choose a Sentence Tranformer model that is suitable for asymmetric semantic search (short query and longer passages), according to [documentation](https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:56:25.360959Z",
+     "iopub.status.busy": "2022-08-21T08:56:25.360546Z",
+     "iopub.status.idle": "2022-08-21T08:58:07.214654Z",
+     "shell.execute_reply": "2022-08-21T08:58:07.213653Z",
+     "shell.execute_reply.started": "2022-08-21T08:56:25.360926Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "from haystack.nodes import EmbeddingRetriever\n",
+    "\n",
+    "retriever = EmbeddingRetriever(\n",
+    "    document_store=document_store,\n",
+    "    embedding_model=\"sentence-transformers/msmarco-distilbert-base-tas-b\",\n",
+    "    model_format=\"sentence_transformers\",\n",
+    "    embed_meta_fields=[\"name\"],\n",
+    ")\n",
+    "\n",
+    "# generate embeddings\n",
+    "document_store.update_embeddings(retriever)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save and export index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import shutil\n",
+    "import glob"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 73,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-08-21T08:58:33.494417Z",
+     "iopub.status.busy": "2022-08-21T08:58:33.493822Z",
+     "iopub.status.idle": "2022-08-21T08:58:33.635915Z",
+     "shell.execute_reply": "2022-08-21T08:58:33.634599Z",
+     "shell.execute_reply.started": "2022-08-21T08:58:33.494382Z"
+    },
+    "trusted": true
+   },
+   "outputs": [],
+   "source": [
+    "OUT_DIR = \"YOUR-OUT-DIR\"\n",
+    "\n",
+    "document_store.save(\"my_faiss_index.faiss\")\n",
+    "for f in glob.glob(\"*faiss*.*\") + glob.glob(\"faiss*.*\"):\n",
+    "    shutil.copy(f, OUT_DIR)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.7.13 ('venv': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.13"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "c114177cb475e38b99e396ae1ef7cfcaaa7967120589f47745b82f90d7e35d1b"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

pages/Info.py

@@ -0,0 +1,10 @@
+import streamlit as st
+
+from app_utils.frontend_utils import build_sidebar
+
+build_sidebar()
+
+with open("README.md", "r") as fin:
+    readme = fin.read().rpartition("---")[-1]
+
+st.markdown(readme, unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,6 @@
+farm-haystack==1.7.1
+faiss-cpu<=1.7.2
+plotly==5.10.0
+# commented to not interfere with streamlit SDK in HF spces
+# uncomment for local installation
+#streamlit==1.10.0