Upload 16 files

Dockerfile

@@ -0,0 +1,42 @@
+ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
+FROM nvidia/cuda:${CUDA_IMAGE}
+
+# Install Python 3.10 and other necessary packages
+RUN apt-get update && apt-get install -y \
+    python3.10 python3-pip \
+    tesseract-ocr \
+    libtesseract-dev \
+    libgl1-mesa-glx \
+    poppler-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the working directory in the container to /app
+WORKDIR /app
+
+# Copy the requirements file into the container at /app
+COPY requirements.txt /app/
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of your application's code
+COPY . /app
+
+# Create a user to run the application
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory in the user's home directory
+WORKDIR $HOME/app
+COPY --chown=user . $HOME/app
+
+# Make port 8000 available to the world outside this container
+EXPOSE 8000
+
+# Define environment variable
+ENV NAME World
+
+# Run the application when the container launches
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

RAG_utils.py

@@ -0,0 +1,1001 @@
+import os
+import re
+import json
+import torch
+
+import openai
+import logging
+import asyncio
+import aiohttp
+import pandas as pd
+import numpy as np
+import evaluate
+import qdrant_client
+from pydantic import BaseModel, Field
+from typing import Any, List, Tuple, Set, Dict, Optional, Union
+from sklearn.metrics.pairwise import cosine_similarity
+
+from unstructured.partition.pdf import partition_pdf
+
+import llama_index
+from llama_index import PromptTemplate
+from llama_index.retrievers import VectorIndexRetriever, BaseRetriever, BM25Retriever
+from llama_index.query_engine import RetrieverQueryEngine
+from llama_index import get_response_synthesizer
+from llama_index.schema import NodeWithScore
+from llama_index.query_engine import RetrieverQueryEngine
+from llama_index import VectorStoreIndex, ServiceContext
+from llama_index.embeddings import OpenAIEmbedding
+from llama_index.llms import HuggingFaceLLM
+import requests
+from llama_index.llms import (
+    CustomLLM,
+    CompletionResponse,
+    CompletionResponseGen,
+    LLMMetadata,
+)
+from llama_index.query_engine import RetrieverQueryEngine
+from llama_index.llms.base import llm_completion_callback
+from llama_index.vector_stores.qdrant import QdrantVectorStore
+from llama_index.storage.storage_context import StorageContext
+from llama_index.postprocessor import SentenceTransformerRerank, LLMRerank
+
+from tempfile import NamedTemporaryFile
+# Configure basic logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Create a logger object
+logger = logging.getLogger(__name__)
+
+class ConfigManager:
+    """
+    A class to manage loading and accessing configuration settings.
+
+    Attributes:
+        config (dict): Dictionary to hold configuration settings.
+
+    Methods:
+        load_config(config_path: str): Loads the configuration from a given JSON file.
+        get_config_value(key: str): Retrieves a specific configuration value.
+    """
+
+    def __init__(self):
+        self.configs = {}
+        
+    def load_config(self, config_name: str, config_path: str) -> None:
+        """
+        Loads configuration settings from a specified JSON file into a named configuration.
+
+        Args:
+            config_name (str): The name to assign to this set of configurations.
+            config_path (str): The path to the configuration file.
+
+        Raises:
+            FileNotFoundError: If the config file is not found.
+            json.JSONDecodeError: If there is an error parsing the config file.
+        """
+        try:
+            with open(config_path, 'r') as f:
+                self.configs[config_name] = json.load(f)
+        except FileNotFoundError:
+            logging.error(f"Config file not found at {config_path}")
+            raise
+        except json.JSONDecodeError as e:
+            logging.error(f"Error decoding config file: {e}")
+            raise
+
+
+    def get_config_value(self, config_name: str, key: str) -> str:
+        """
+        Retrieves a specific configuration value.
+
+        Args:
+            key (str): The key for the configuration setting.
+
+        Returns:
+            str: The value of the configuration setting.
+
+        Raises:
+            ValueError: If the key is not found or is set to a placeholder value.
+        """
+        value = self.configs.get(config_name, {}).get(key)
+        if value is None or value == "ENTER_YOUR_TOKEN_HERE":
+            raise ValueError(f"Please set your '{key}' in the config.json file.")
+        return value
+
+class base_utils:
+    """
+    A utility class providing miscellaneous static methods for processing and analyzing text data,
+    particularly from PDF documents and filenames. This class also includes methods for file operations.
+
+    This class encapsulates the functionality of extracting key information from text, such as scores,
+    reasoning, and IDs, locating specific data within a DataFrame based on an ID extracted from a filename,
+    and reading content from files.
+
+    Attributes:
+        None (This class contains only static methods and does not maintain any state)
+
+    Methods:
+        extract_score_reasoning(text: str) -> Dict[str, Optional[str]]:
+            Extracts a score and reasoning from a given text using regular expressions.
+
+        extract_id_from_filename(filename: str) -> Optional[int]:
+            Extracts an ID from a given filename based on a specified pattern.
+
+        find_row_for_pdf(pdf_filename: str, dataframe: pd.DataFrame) -> Union[pd.Series, str]:
+            Searches for a row in a DataFrame that matches an ID extracted from a PDF filename.
+
+        read_from_file(file_path: str) -> str:
+            Reads the content of a file and returns it as a string.
+    """
+    
+    @staticmethod
+    def read_from_file(file_path: str) -> str:
+        """
+        Reads the content of a file and returns it as a string.
+
+        Args:
+            file_path (str): The path to the file to be read.
+
+        Returns:
+            str: The content of the file.
+        """
+        with open(file_path, 'r') as prompt_file:
+            prompt = prompt_file.read()
+        return prompt
+        
+    @staticmethod
+    def extract_id_from_filename(filename: str) -> Optional[int]:
+        """
+        Extracts an ID from a filename, assuming a specific format ('Id_{I}.pdf', where {I} is the ID).
+
+        Args:
+            filename (str): The filename from which to extract the ID.
+
+        Returns:
+            int: The extracted ID as an integer, or None if the pattern is not found.
+        """
+        # Assuming the file name is in the format 'Id_{I}.pdf', where {I} is the ID
+        match = re.search(r'Id_(\d+).pdf', filename)
+        if match:
+            return int(match.group(1))  # Convert to integer if ID is numeric
+        else:
+            return None
+    
+    @staticmethod
+    def extract_score_reasoning(text: str) -> Dict[str, Optional[str]]:
+        """
+        Extracts score and reasoning from a given text using regular expressions.
+
+        Args:
+            text (str): The text from which to extract the score and reasoning.
+
+        Returns:
+            dict: A dictionary containing 'score' and 'reasoning', extracted from the text.
+        """
+        # Define regular expression patterns for score and reasoning
+        score_pattern = r"Score: (\d+)"
+        reasoning_pattern = r"Reasoning: (.+)"
+        
+        # Extract data using regular expressions
+        score_match = re.search(score_pattern, text)
+        reasoning_match = re.search(reasoning_pattern, text, re.DOTALL)  # re.DOTALL allows '.' to match newlines
+        
+        # Extract and return the results
+        extracted_data = {
+            "score": score_match.group(1) if score_match else None,
+            "reasoning": reasoning_match.group(1).strip() if reasoning_match else None
+        }
+        
+        return extracted_data
+            
+            
+    @staticmethod
+    def find_row_for_pdf(pdf_filename: str, dataframe: pd.DataFrame) -> Union[pd.Series, str]:
+        """
+        Finds the row in a dataframe corresponding to the ID extracted from a given PDF filename.
+
+        Args:
+            pdf_filename (str): The filename of the PDF.
+            dataframe (pandas.DataFrame): The dataframe in which to find the corresponding row.
+
+        Returns:
+            pandas.Series or str: The matched row from the dataframe or a message indicating
+                                  that no matching row or invalid filename was found.
+        """
+        pdf_id = Utility.extract_id_from_filename(pdf_filename)
+        if pdf_id is not None:
+            # Assuming the first column contains the ID
+            matched_row = dataframe[dataframe.iloc[:, 0] == pdf_id]
+            if not matched_row.empty:
+                return matched_row
+            else:
+                return "No matching row found."
+        else:
+            return "Invalid file name."
+            
+            
+class PDFProcessor_Unstructured:
+    """
+    A class to process PDF files, providing functionalities for extracting, categorizing,
+    and merging elements from a PDF file.
+
+    This class is designed to handle unstructured PDF documents, particularly useful for
+    tasks involving text extraction, categorization, and data processing within PDFs.
+
+    Attributes:
+        file_path (str): The full path to the PDF file.
+        folder_path (str): The directory path where the PDF file is located.
+        file_name (str): The name of the PDF file.
+        texts (List[str]): A list to store extracted text chunks.
+        tables (List[str]): A list to store extracted tables.
+        
+
+    Methods:
+        extract_pdf_elements() -> List:
+            Extracts images, tables, and text chunks from a PDF file.
+
+        categorize_elements(raw_pdf_elements: List) -> None:
+            Categorizes extracted elements from a PDF into tables and texts.
+
+        merge_chunks() -> List[str]:
+            Merges text chunks based on punctuation and character case criteria.
+
+        should_skip_chunk(chunk: str) -> bool:
+            Determines if a chunk should be skipped based on its content.
+
+        should_merge_with_next(current_chunk: str, next_chunk: str) -> bool:
+            Determines if the current chunk should be merged with the next one.
+
+        process_pdf() -> Tuple[List[str], List[str]]:
+            Processes the PDF by extracting, categorizing, and merging elements.
+
+        process_pdf_file(uploaded_file) -> Tuple[List[str], List[str]]:
+            Processes an uploaded PDF file to extract and categorize text and tables.
+    """
+
+    def __init__(self, config: Dict[str, any]):
+        self.file_path = None
+        self.folder_path = None
+        self.file_name = None
+        self.texts = []
+        self.tables = []
+        self.config = config if config is not None else self.default_config()
+        logger.info(f"Initialized PdfProcessor_Unstructured for file: {self.file_name}")
+        
+    @staticmethod
+    def default_config() -> Dict[str, any]:
+        """
+        Returns the default configuration for PDF processing.
+
+        Returns:
+            Dict[str, any]: Default configuration options.
+        """
+        return {
+            "extract_images": False,
+            "infer_table_structure": True,
+            "chunking_strategy": "by_title",
+            "max_characters": 10000,
+            "combine_text_under_n_chars": 100,
+            "strategy": "fast",
+            "model_name": "yolox"
+        }
+
+
+    def extract_pdf_elements(self) -> List:
+        """
+        Extracts images, tables, and text chunks from a PDF file.
+
+        Returns:
+            List: A list of extracted elements from the PDF.
+        """
+        logger.info("Starting extraction of PDF elements.")
+        try:
+            extracted_elements = partition_pdf(
+                filename=self.file_path,
+                extract_images_in_pdf=False,
+                infer_table_structure=True,
+                chunking_strategy="by_title",
+                strategy = "fast",
+                max_characters=10000,
+                combine_text_under_n_chars=100,
+                image_output_dir_path=self.folder_path,
+            )
+            logger.info("Extraction of PDF elements completed successfully.")
+            return extracted_elements
+        except Exception as e:
+            logger.error(f"Error extracting PDF elements: {e}", exc_info=True)
+            raise
+
+    def categorize_elements(self, raw_pdf_elements: List) -> None:
+        """
+        Categorizes extracted elements from a PDF into tables and texts.
+
+        Args:
+            raw_pdf_elements (List): A list of elements extracted from the PDF.
+        """
+        logger.debug("Starting categorization of PDF elements.")
+        for element in raw_pdf_elements:
+            element_type = str(type(element))
+            if "unstructured.documents.elements.Table" in element_type:
+                self.tables.append(str(element))
+            elif "unstructured.documents.elements.CompositeElement" in element_type:
+                self.texts.append(str(element))
+                
+        logger.debug("Categorization of PDF elements completed.")
+
+    def merge_chunks(self) -> List[str]:
+        """
+        Merges text chunks based on punctuation and character case criteria.
+
+        Returns:
+            List[str]: A list of merged text chunks.
+        """
+        logger.debug("Starting merging of text chunks.")
+        
+        merged_chunks = []
+        skip_next = False
+
+        for i, current_chunk in enumerate(self.texts[:-1]):
+            next_chunk = self.texts[i + 1]
+
+            if self.should_skip_chunk(current_chunk):
+                continue
+
+            if self.should_merge_with_next(current_chunk, next_chunk):
+                merged_chunks.append(current_chunk + " " + next_chunk)
+                skip_next = True
+            else:
+                merged_chunks.append(current_chunk)
+
+        if not skip_next:
+            merged_chunks.append(self.texts[-1])
+        
+        logger.debug("Merging of text chunks completed.")
+
+        return merged_chunks
+
+    @staticmethod
+    def should_skip_chunk(chunk: str) -> bool:
+        """
+        Determines if a chunk should be skipped based on its content.
+
+        Args:
+            chunk (str): The text chunk to be evaluated.
+
+        Returns:
+            bool: True if the chunk should be skipped, False otherwise.
+        """
+        return (chunk.lower().startswith(("figure", "fig", "table")) or
+                not chunk[0].isalnum() or
+                re.match(r'^\d+\.', chunk))
+
+    @staticmethod
+    def should_merge_with_next(current_chunk: str, next_chunk: str) -> bool:
+        """
+        Determines if the current chunk should be merged with the next one.
+
+        Args:
+            current_chunk (str): The current text chunk.
+            next_chunk (str): The next text chunk.
+
+        Returns:
+            bool: True if the chunks should be merged, False otherwise.
+        """
+        return (current_chunk.endswith(",") or
+                (current_chunk[-1].islower() and next_chunk[0].islower()))
+
+    def process_pdf(self) -> Tuple[List[str], List[str]]:
+        """
+        Processes the PDF by extracting, categorizing, and merging elements.
+
+        Returns:
+            Tuple[List[str], List[str]]: A tuple of merged text chunks and tables.
+        """
+        logger.info("Starting processing of the PDF.")
+        try:
+            raw_pdf_elements = self.extract_pdf_elements()
+            self.categorize_elements(raw_pdf_elements)
+            merged_chunks = self.merge_chunks()
+            return merged_chunks, self.tables
+        except Exception as e:
+            logger.error(f"Error processing PDF: {e}", exc_info=True)
+            raise
+            
+    def process_pdf_file(self, uploaded_file):
+        """
+        Process an uploaded PDF file.
+
+        If a new file is uploaded, the previously stored file is deleted.
+        The method updates the file path, processes the PDF, and returns the results.
+        
+        Parameters:
+        uploaded_file: The new PDF file uploaded for processing.
+
+        Returns:
+        The results of processing the PDF file.
+        """
+        # Delete the previous file if it exists
+        if self.file_path and os.path.exists(self.file_path):
+            try:
+                os.remove(self.file_path)
+                logging.debug(f"Previous file {self.file_path} deleted.")
+            except Exception as e:
+                logging.warning(f"Error deleting previous file: {e}", exc_info=True)
+
+        # Process the new file
+        self.file_path = str(uploaded_file)
+        self.folder_path = os.path.dirname(self.file_path)
+        logging.info(f"Starting to process the PDF file: {self.file_path}")
+
+        try:
+            logging.debug(f"Processing PDF at {self.file_path}")
+            results = self.process_pdf()  # Assuming this is a defined method
+            logging.info("PDF processing completed successfully.")
+            return results
+        except Exception as e:
+            logging.error(f"Error processing PDF file: {e}", exc_info=True)
+            raise
+
+
+class HybridRetriever(BaseRetriever):
+    """
+    A hybrid retriever that combines results from vector-based and BM25 retrieval methods.
+    Inherits from BaseRetriever.
+    
+    This class uses two different retrieval methods and merges their results to provide a
+    comprehensive set of documents in response to a query. It ensures diversity in the
+    retrieved documents by leveraging the strengths of both retrieval methods.
+
+    Attributes:
+        vector_retriever: An instance of a vector-based retriever.
+        bm25_retriever: An instance of a BM25 retriever.
+
+    Methods:
+        __init__(vector_retriever, bm25_retriever): Initializes the HybridRetriever with vector and BM25 retrievers.
+        _retrieve(query, **kwargs): Performs the retrieval operation by combining results from both retrievers.
+        _combine_results(bm25_nodes, vector_nodes): Combines and de-duplicates the results from both retrievers.
+    """
+
+    def __init__(self, vector_retriever, bm25_retriever):
+        super().__init__()
+        self.vector_retriever = vector_retriever
+        self.bm25_retriever = bm25_retriever
+        logger.info("HybridRetriever initialized with vector and BM25 retrievers.")
+
+    def _retrieve(self, query: str, **kwargs) -> List:
+        """
+        Retrieves and combines results from both vector and BM25 retrievers.
+
+        Args:
+            query: The query string for document retrieval.
+            **kwargs: Additional keyword arguments for retrieval.
+
+        Returns:
+            List: Combined list of unique nodes retrieved from both methods.
+        """
+        logger.info(f"Retrieving documents for query: {query}")
+        try:
+            bm25_nodes = self.bm25_retriever.retrieve(query, **kwargs)
+            vector_nodes = self.vector_retriever.retrieve(query, **kwargs)
+            combined_nodes = self._combine_results(bm25_nodes, vector_nodes)
+
+            logger.info(f"Retrieved {len(combined_nodes)} unique nodes combining vector and BM25 retrievers.")
+            return combined_nodes
+        except Exception as e:
+            logger.error(f"Error in retrieval: {e}")
+            raise
+
+    @staticmethod
+    def _combine_results(bm25_nodes: List, vector_nodes: List) -> List:
+        """
+        Combines and de-duplicates results from BM25 and vector retrievers.
+
+        Args:
+            bm25_nodes: Nodes retrieved from BM25 retriever.
+            vector_nodes: Nodes retrieved from vector retriever.
+
+        Returns:
+            List: Combined list of unique nodes.
+        """
+        node_ids: Set = set()
+        combined_nodes = []
+
+        for node in bm25_nodes + vector_nodes:
+            if node.node_id not in node_ids:
+                combined_nodes.append(node)
+                node_ids.add(node.node_id)
+
+        return combined_nodes
+
+        
+
+class PDFQueryEngine:
+    """
+    A class to handle the process of setting up a query engine and performing queries on PDF documents.
+
+    This class encapsulates the functionality of creating prompt templates, embedding models, service contexts,
+    indexes, hybrid retrievers, response synthesizers, and executing queries on the set up engine.
+
+    Attributes:
+        documents (List): A list of documents to be indexed.
+        llm (Language Model): The language model to be used for embeddings and queries.
+        qa_prompt_tmpl (str): Template for creating query prompts.
+        queries (List[str]): List of queries to be executed.
+
+    Methods:
+        setup_query_engine(): Sets up the query engine with all necessary components.
+        execute_queries(): Executes the predefined queries and prints the results.
+    """
+
+    def __init__(self, documents: List[Any], llm: Any, embed_model: Any, qa_prompt_tmpl: Any):
+        
+        self.documents = documents
+        self.llm = llm
+        self.embed_model = embed_model
+        self.qa_prompt_tmpl = qa_prompt_tmpl
+        self.base_utils = base_utils()
+        self.config_manager = ConfigManager()
+        
+        
+    
+        logger.info("PDFQueryEngine initialized.")
+
+    def format_example(self, example):
+        """
+        Formats a few-shot example into a string.
+
+        Args:
+            example (dict): A dictionary containing 'query', 'score', and 'reasoning' for the few-shot example.
+
+        Returns:
+            str: Formatted few-shot example text.
+        """
+        return "Example:\nQuery: {}\nScore: {}\nReasoning: {}\n".format(
+            example['query'], example['score'], example['reasoning']
+        )
+
+
+    def setup_query_engine(self):
+        """
+        Sets up the query engine by initializing and configuring the embedding model, service context, index,
+        hybrid retriever (combining vector and BM25 retrievers), and the response synthesizer.
+
+        Args:
+            embed_model: The embedding model to be used.
+            service_context: The context for providing services to the query engine.
+            index: The index used for storing and retrieving documents.
+            hybrid_retriever: The retriever that combines vector and BM25 retrieval methods.
+            response_synthesizer: The synthesizer for generating responses to queries.
+
+        Returns:
+            Any: The configured query engine.
+        """
+        client = qdrant_client.QdrantClient(
+            # you can use :memory: mode for fast and light-weight experiments,
+            # it does not require to have Qdrant deployed anywhere
+            # but requires qdrant-client >= 1.1.1
+            location=":memory:"
+            # otherwise set Qdrant instance address with:
+            # uri="http://<host>:<port>"
+            # set API KEY for Qdrant Cloud
+            # api_key="<qdrant-api-key>",
+            )
+        try:
+            logger.info("Initializing the service context for query engine setup.")
+            service_context = ServiceContext.from_defaults(llm=self.llm, embed_model=self.embed_model)
+            vector_store = QdrantVectorStore(client=client, collection_name="med_library")
+            storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+            logger.info("Creating an index from documents.")
+            index = VectorStoreIndex.from_documents(documents=self.documents, storage_context=storage_context, service_context=service_context)
+            nodes = service_context.node_parser.get_nodes_from_documents(self.documents)
+
+            logger.info("Setting up vector and BM25 retrievers.")
+            vector_retriever = index.as_retriever(similarity_top_k=3)
+            bm25_retriever = BM25Retriever.from_defaults(nodes=nodes, similarity_top_k=3)
+            hybrid_retriever = HybridRetriever(vector_retriever, bm25_retriever)
+        
+            logger.info("Configuring the response synthesizer with the prompt template.")
+            qa_prompt = PromptTemplate(self.qa_prompt_tmpl)
+            response_synthesizer = get_response_synthesizer(
+                service_context=service_context,
+                text_qa_template=qa_prompt,
+                response_mode="compact",
+            )
+
+            logger.info("Assembling the query engine with reranker and synthesizer.")
+            reranker = SentenceTransformerRerank(top_n=3, model="BAAI/bge-reranker-base") 
+            query_engine = RetrieverQueryEngine.from_args(
+                retriever=hybrid_retriever,
+                node_postprocessors=[reranker],
+                response_synthesizer=response_synthesizer,
+            )
+
+            logger.info("Query engine setup complete.")
+            return query_engine
+        except Exception as e:
+            logger.error(f"Error during query engine setup: {e}")
+            raise
+
+    # def evaluate_with_llm(self, reg_result: Any, peer_result: Any, guidelines_result: Any, queries: List[str]) -> Tuple[int, List[int], int, float, List[str]]:
+    #     """
+    #     Evaluate documents using a language model based on various criteria.
+    #     Args:
+    #         reg_result (Any): Result related to registration.
+    #         peer_result (Any): Result related to peer review.
+    #         guidelines_result (Any): Result related to following guidelines.
+    #         queries (List[str]): A list of queries to be processed.
+    #     Returns:
+    #         Tuple[int, List[int], int, float, List[str]]: A tuple containing the total score, a list of scores per criteria.
+    #     """
+    
+    #     logger.info("Starting evaluation with LLM.")
+    #     self.config_manager.load_config("few_shot", "few_shot.json")
+    #     query_engine = self.setup_query_engine()
+
+    #     total_score = 0
+    #     criteria_met = 0
+    #     reasoning = []
+    
+    #     for j, query in enumerate(queries):
+    #         # Handle special cases based on the value of j and other conditions
+    #         if j == 1 and reg_result:
+    #             extracted_data = {"score": 1, "reasoning": reg_result[0]}
+    #         elif j == 2 and guidelines_result:
+    #             extracted_data = {"score": 1, "reasoning": "The article is published in a journal following EQUATOR-NETWORK reporting guidelines"}
+    #         elif j == 8 and (guidelines_result or peer_result):
+    #             extracted_data = {"score": 1, "reasoning": "The article is published in a peer-reviewed journal."}
+    #         else:
+                
+    #             # Execute the query
+    #             result = query_engine.query(query).response
+    #             extracted_data = self.base_utils.extract_score_reasoning(result)
+
+
+    #         # Validate and accumulate the scores
+    #         extracted_data_score = 0 if extracted_data.get("score") is None else int(extracted_data.get("score"))
+    #         if extracted_data_score > 0:
+    #             criteria_met += 1
+    #         reasoning.append(extracted_data["reasoning"])
+    #         total_score += extracted_data_score
+
+    #     score_percentage = (float(total_score) / len(queries)) * 100
+    #     logger.info("Evaluation completed.")
+    #     return total_score, criteria_met, score_percentage, reasoning
+        
+    async def evaluate_with_llm_async(self, reg_result: Any, peer_result: Any, guidelines_result: Any, queries: List[str]) -> Tuple[int, List[int], int, float, List[str]]:
+        logger.info("Starting evaluation with LLM.")
+        self.config_manager.load_config("few_shot", "few_shot.json")
+        # Setup your query engine, if it's using aiohttp this is where you'd configure it
+
+        total_score = 0
+        criteria_met = 0
+        reasoning = []
+
+        async def handle_query(session, j, query):
+            if j == 1 and reg_result:
+                return {"score": 1, "reasoning": reg_result[0]}
+            elif j == 2 and guidelines_result:
+                return {"score": 1, "reasoning": "The article is published in a journal following EQUATOR-NETWORK reporting guidelines"}
+            elif j == 8 and (guidelines_result or peer_result):
+                return {"score": 1, "reasoning": "The article is published in a peer-reviewed journal."}
+            else:
+                # Here, adapt your query engine or direct API call to use aiohttp
+                async with session.post('Your API Endpoint', json={'query': query}) as response:
+                    result = await response.json()
+                    return self.base_utils.extract_score_reasoning(result)
+
+        async with aiohttp.ClientSession() as session:
+            tasks = [handle_query(session, j, query) for j, query in enumerate(queries)]
+            results = await asyncio.gather(*tasks)
+
+        # Process results
+        for extracted_data in results:
+            extracted_data_score = 0 if extracted_data.get("score") is None else int(extracted_data.get("score"))
+            if extracted_data_score > 0:
+                criteria_met += 1
+            reasoning.append(extracted_data["reasoning"])
+            total_score += extracted_data_score
+
+        score_percentage = (float(total_score) / len(queries)) * 100
+        logger.info("Evaluation completed.")
+        return total_score, criteria_met, len(queries), score_percentage, reasoning
+
+
+
+class MixtralLLM(CustomLLM):
+    """
+    A custom language model class for interfacing with the Hugging Face API, specifically using the Mixtral model.
+
+    Attributes:
+        context_window (int): Number of tokens used for context during inference.
+        num_output (int): Number of tokens to generate as output.
+        temperature (float): Sampling temperature for token generation.
+        model_name (str): Name of the model on Hugging Face's model hub.
+        api_key (str): API key for authenticating with the Hugging Face API.
+
+    Methods:
+        metadata: Retrieves metadata about the model.
+        do_hf_call: Makes an API call to the Hugging Face model.
+        complete: Generates a complete response for a given prompt.
+        stream_complete: Streams a series of token completions for a given prompt.
+    """
+    context_window: int = Field(..., description="Number of tokens used for context during inference.")
+    num_output: int = Field(..., description="Number of tokens to generate as output.")
+    temperature: float = Field(..., description="Sampling temperature for token generation.")
+    model_name: str = Field(..., description="Name of the model on Hugging Face's model hub.")
+    api_key: str = Field(..., description="API key for authenticating with the Hugging Face API.")
+
+
+    @property
+    def metadata(self) -> LLMMetadata:
+        """
+        Retrieves metadata for the Mixtral LLM.
+
+        Returns:
+            LLMMetadata: An object containing metadata such as context window, number of outputs, and model name.
+        """
+        return LLMMetadata(
+            context_window=self.context_window,
+            num_output=self.num_output,
+            model_name=self.model_name,
+        )
+
+    def do_hf_call(self, prompt: str) -> str:
+        """
+        Makes an API call to the Hugging Face model and retrieves the generated response.
+
+        Args:
+            prompt (str): The input prompt for the model.
+
+        Returns:
+            str: The text generated by the model in response to the prompt.
+        
+        Raises:
+            Exception: If the API call fails or returns an error.
+        """
+        data = {
+            "inputs": prompt,
+            "parameters": {"Temperature": self.temperature}
+        }
+
+        # Makes a POST request to the Hugging Face API to get the model's response
+        response = requests.post(
+            f'https://api-inference.huggingface.co/models/{self.model_name}',
+            headers={
+                'authorization': f'Bearer {self.api_key}',
+                'content-type': 'application/json',
+            },
+            json=data,
+            stream=True
+        )
+
+        # Checks for a successful response and parses the generated text
+        if response.status_code != 200 or not response.json() or 'error' in response.json():
+            print(f"Error: {response}")
+            return "Unable to answer for technical reasons."
+        full_txt = response.json()[0]['generated_text']
+        # Finds the section of the text following the context separator
+        offset = full_txt.find("---------------------")
+        ss = full_txt[offset:]
+        # Extracts the actual answer from the response
+        offset = ss.find("Answer:")
+        return ss[offset+7:].strip()
+
+    
+    @llm_completion_callback()
+    def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse:
+        """
+        Generates a complete response for a given prompt using the Hugging Face API.
+
+        Args:
+            prompt (str): The input prompt for the model.
+            **kwargs: Additional keyword arguments for the completion.
+
+        Returns:
+            CompletionResponse: The complete response from the model.
+        """
+        response = self.do_hf_call(prompt)
+        return CompletionResponse(text=response)
+
+    
+    @llm_completion_callback()
+    def stream_complete(
+            self, prompt: str, **kwargs: Any
+    ) -> CompletionResponseGen:
+        """
+        Streams a series of token completions as a response for the given prompt.
+
+        This method is useful for streaming responses where each token is generated sequentially.
+
+        Args:
+            prompt (str): The input prompt for the model.
+            **kwargs: Additional keyword arguments for the streaming completion.
+
+        Yields:
+            CompletionResponseGen: A generator yielding each token in the completion response.
+        """
+        # Yields a stream of tokens as the completion response for the given prompt
+        response = ""
+        for token in self.do_hf_call(prompt):
+            response += token
+            yield CompletionResponse(text=response, delta=token)
+
+
+
+class KeywordSearch():
+    def __init__(self, chunks):
+        self.chunks = chunks
+    
+    def find_journal_name(self, response: str, journal_list: list) -> str:
+        """
+        Searches for a journal name in a given response string.
+
+        This function iterates through a list of known journal names and checks if any of these
+        names are present in the response string. It returns the first journal name found in the
+        response. If no journal names from the list are found in the response, a default message
+        indicating that the journal name was not found is returned.
+
+        Args:
+            response (str): The response string to search for a journal name.
+            journal_list (list): A list of journal names to search within the response.
+
+        Returns:
+            str: The first journal name found in the response, or a default message if no journal name is found.
+        """
+        response_lower = response.lower()
+        for journal in journal_list:
+            journal_lower = journal.lower()
+            
+            if journal_lower in response_lower:
+                print(journal_lower,response_lower)
+                return True
+        
+        return False
+
+    def check_registration(self):
+        """
+        Check chunks of text for various registration numbers or URLs of registries.
+        Returns the sentence containing a registration number, or if not found,
+        returns chunks containing registry URLs.
+
+        Args:
+        chunks (list of str): List of text chunks to search.
+
+        Returns:
+        list of str: List of matching sentences or chunks, or an empty list if no matches are found.
+        """
+
+        # Patterns for different registration types
+        patterns = {
+            "NCT": r"\(?(NCT#?\s*(No\s*)?)(\d{8})\)?",
+            "ISRCTN": r"(ISRCTN\d{8})",
+            "EudraCT": r"(\d{4}-\d{6}-\d{2})",
+            "UMIN-CTR": r"(UMIN\d{9})",
+            "CTRI": r"(CTRI/\d{4}/\d{2}/\d{6})"
+        }
+
+        # Registry URLs
+        registry_urls = [
+            "www.anzctr.org.au",
+            "anzctr.org.au",
+            "www.clinicaltrials.gov",
+            "clinicaltrials.gov",
+            "www.ISRCTN.org",
+            "ISRCTN.org",
+            "www.umin.ac.jp/ctr/index/htm",
+            "umin.ac.jp/ctr/index/htm",
+            "www.onderzoekmetmensen.nl/en",
+            "onderzoekmetmensen.nl/en",
+            "eudract.ema.europa.eu",
+            "www.eudract.ema.europa.eu"
+            ]
+    
+
+        # Check each chunk for registration numbers
+        for chunk in self.chunks:
+            # Split chunk into sentences
+            sentences = re.split(r'(?<=[.!?]) +', chunk)
+
+            # Check each sentence for any registration number
+            for sentence in sentences:
+                for pattern in patterns.values():
+                    if re.search(pattern, sentence):
+                        return [sentence]  # Return immediately if a registration number is found
+
+        # If no registration number found, check for URLs in chunks
+        matching_chunks = []
+        for chunk in self.chunks:
+            if any(url in chunk for url in registry_urls):
+                matching_chunks.append(chunk)
+
+        return matching_chunks
+
+    
+
+class StringExtraction():
+
+    """
+    A class to handle the the process of extraction of query string from complete LLM responses.
+
+    This class encapsulates the functionality of extracting original ground truth from a labelled data csv and query strings from responses. Please note that
+    LLMs may generate different formatted answers based on different models or different prompting technique. In such cases, extract_original_prompt may not give 
+    satisfactory results. Best case scenario will be write your own string extraction method in such cases.
+
+    
+    Methods:
+        extract_original_prompt(): 
+        extraction_ground_truth(): 
+    """
+
+    def extract_original_prompt(self,result):
+        r1 = result.response.strip().split("\n")
+        binary_response = ""
+        explanation_response = ""
+        for r in r1:
+            if binary_response == "" and (r.find("Yes") >= 0 or r.find("No") >= 0):
+                binary_response = r
+            elif r.find("Reasoning:") >= 0:
+                cut = r.find(":")
+                explanation_response += r[cut+1:].strip()
+            
+        return binary_response,explanation_response
+
+    def extraction_ground_truth(self,paper_name,labelled_data):
+        id = int(paper_name[paper_name.find("_")+1:paper_name.find(".pdf")])
+        id_row = labelled_data[labelled_data["id"] == id]
+        ground_truth = id_row.iloc[:,2:11].values.tolist()[0]
+        binary_ground_truth = []
+        explanation_ground_truth = []
+        for g in ground_truth:
+            if len(g) > 0:
+                binary_ground_truth.append("Yes")
+                explanation_ground_truth.append(g)
+            else:
+                binary_ground_truth.append("No")
+                explanation_ground_truth.append("The article does not provide any relevant information.")
+        return binary_ground_truth,explanation_ground_truth
+    
+
+  
+class EvaluationMetrics():
+    """
+
+    This class encapsulates the evaluation methods that have been used in the project.
+
+    Attributes:
+        explanation_response = a list of detailed response from the LLM model corresponding to each query
+        explanation_ground_truth = the list of ground truth corresponding to each query
+
+    Methods:
+        metric_cosine_similairty(): Sets up the query engine with all necessary components.
+        metric_rouge(): Executes the predefined queries and prints the results.
+        metric_binary_accuracy():
+    """
+
+
+    def __init__(self,explanation_response,explanation_ground_truth,embedding_model):
+        self.explanation_response = explanation_response
+        self.explanation_ground_truth = explanation_ground_truth
+        self.embedding_model = embedding_model
+    
+    def metric_cosine_similarity(self):
+        ground_truth_embedding = self.embedding_model.encode(self.explanation_ground_truth)
+        explanation_response_embedding = self.embedding_model.encode(self.explanation_response)
+        return np.diag(cosine_similarity(ground_truth_embedding,explanation_response_embedding))
+    
+    def metric_rouge(self):
+        rouge = evaluate.load("rouge")
+        results = rouge.compute(predictions = self.explanation_response,references = self.explanation_ground_truth)
+        return results 
+
+    def binary_accuracy(self,binary_response,binary_ground_truth):
+        count = 0
+        if len(binary_response) != len(binary_ground_truth):
+            return "Arrays which are to be compared has different lengths."
+        else:
+            for i in range(len(binary_response)):
+                if binary_response[i] == binary_ground_truth[i]:
+                    count += 1
+            return np.round(count/len(binary_response),2)

README.md

@@ -1,11 +1,12 @@
 ---
 title: Epitomea Demo
-emoji: 😻
-colorFrom: blue
-colorTo: blue
+emoji: 🐨
+colorFrom: purple
+colorTo: indigo
 sdk: docker
 pinned: false
 license: apache-2.0
+startup_duration_timeout: 1h
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,364 @@
+import os
+import re
+import json
+import getpass
+import logging
+import openai
+import asyncio
+from typing import Any, List, Tuple, Dict
+import gradio as gr
+import llama_index
+from llama_index import Document
+from llama_index.llms import OpenAI
+from llama_index.embeddings import OpenAIEmbedding, HuggingFaceEmbedding
+from llama_index.llms import HuggingFaceLLM
+import requests
+
+from RAG_utils import PDFProcessor_Unstructured, PDFQueryEngine, HybridRetriever, MixtralLLM, KeywordSearch, base_utils, ConfigManager
+
+
+# Configure basic logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Create a logger object
+logger = logging.getLogger(__name__)
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+config_manager = ConfigManager()
+#config_manager.load_config("api", "Config/api_config.json")
+config_manager.load_config("model", "model_config.json")
+
+openai.api_key = os.environ['OPENAI_API_KEY'] #config_manager.get_config_value("api", "OPENAI_API_KEY")
+hf_token = os.environ['HF_TOKEN']#config_manager.get_config_value("api", "HF_TOKEN")
+
+# PDF rendering and chunking parameters
+pdf_processing_config = config_manager.get_config_value("model", "pdf_processing")
+
+
+ALLOWED_EXTENSIONS = config_manager.get_config_value("model", "allowed_extensions")
+embed = config_manager.get_config_value("model", "embeddings")
+embed_model_name = config_manager.get_config_value("model", "embeddings_model")
+
+
+#llm_model = config_manager.get_config_value("model", "llm_model")
+model_temperature = config_manager.get_config_value("model", "model_temp")
+output_token_size = config_manager.get_config_value("model", "max_tokens")
+model_context_window = config_manager.get_config_value("model", "context_window")
+
+gpt_prompt_path = config_manager.get_config_value("model","GPT_PROMPT_PATH")
+mistral_prompt_path = config_manager.get_config_value("model","MISTRAL_PROMPT_PATH")
+info_prompt_path = config_manager.get_config_value("model", "INFO_PROMPT_PATH")
+
+peer_review_journals_path = config_manager.get_config_value("model", "peer_review_journals_path")
+eq_network_journals_path = config_manager.get_config_value("model", "eq_network_journals_path")
+
+queries = config_manager.get_config_value("model", "queries")
+criteria = config_manager.get_config_value("model", "criteria")
+num_criteria = len(queries)
+
+author_query = config_manager.get_config_value("model", "author_query")
+journal_query = config_manager.get_config_value("model", "journal_query")
+
+
+# Helper function to check if the file extension is allowed
+def allowed_file(filename):
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+def generate_score_bar(score, num_criteria):
+    # Convert and round the score from a 9-point scale to a 100-point scale
+    score_out_of_100 = round((score / num_criteria) * 100)
+
+    # Determine the color and text based on the original score
+    if score == 9:
+        color = "#4CAF50"  # green
+        text = "Very good"
+    elif score in [7, 8]:
+        color = "#FFEB3B"  # yellow
+        text = "Good"
+    elif score in [5, 6]:
+        color = "#FF9800"  # orange
+        text = "Ok"
+    elif score in [3, 4]:
+        color = "#F44336"  # red
+        text = "Bad"
+    else:  # score < 3
+        color = "#800000"  # maroon
+        text = "Very bad"
+
+    # Create the HTML for the score bar
+    score_bar_html = f"""
+        <div style="background-color: #ddd; border-radius: 10px; position: relative; height: 20px; width: 100%;">
+            <div style="background-color: {color}; height: 100%; border-radius: 10px; width: {score_out_of_100}%;"></div>
+        </div>
+        <p style="color: {color};">{text}</p>  <!-- Display the text -->
+    """
+    return score_bar_html
+def format_example(example):
+        """
+        Formats a few-shot example into a string.
+        Args:
+            example (dict): A dictionary containing 'query', 'score', and 'reasoning' for the few-shot example.
+        Returns:
+            str: Formatted few-shot example text.
+        """
+        return "Example:\nQuery: {}\n Direct Answer: {}\n".format(
+            example['query'], example['Answer'])
+    
+def process_pdf(uploaded_file, llm_model, n_criteria = num_criteria):
+    # Process the PDF file
+    pdf_processor = PDFProcessor_Unstructured(pdf_processing_config)
+    merged_chunks, tables = pdf_processor.process_pdf_file(uploaded_file)
+    documents = [Document(text=t) for t in merged_chunks]
+
+    # Prompts and Queries
+    utils = base_utils()
+    
+    info_prompt = utils.read_from_file(info_prompt_path)
+
+    # LLM Model choice
+    try:
+        if llm_model == "Model 1":
+            llm = OpenAI(model="gpt-4-1106-preview", temperature=model_temperature, max_tokens=output_token_size)
+            general_prompt = utils.read_from_file(gpt_prompt_path)
+
+        elif llm_model == "Model 2":
+            if any(param is None for param in [model_context_window, output_token_size, model_temperature, hf_token]):
+                raise ValueError("All parameters are required for Mistral LLM.")
+
+               
+            llm = MixtralLLM(context_window=model_context_window, num_output=output_token_size,
+                           temperature=model_temperature, model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", api_key=hf_token)
+            general_prompt = utils.read_from_file(mistral_prompt_path)
+        else:
+            raise ValueError(f"Unsupported language model: {llm_model}")
+
+    except Exception as e:
+        logger.error(f"Error initializing language model '{llm_model}': {e}", exc_info=True)
+        raise  # Or handle the exception as needed
+
+    # Embedding model choice for RAG
+    try:
+        if embed == "openai":
+            embed_model = OpenAIEmbedding(model="text-embedding-3-large")
+
+        elif embed == "huggingface":
+            # Use the specified model name
+            embed_model = HuggingFaceEmbedding(embed_model_name)
+        
+        else:
+            raise ValueError(f"Unsupported embedding model: {embed_model}")
+
+
+    except Exception as e:
+        logger.error(f"Error initializing embedding model: {e}", exc_info=True)
+        raise
+
+    
+    peer_review_journals = utils.read_from_file(peer_review_journals_path)
+    eq_network_journals = utils.read_from_file(eq_network_journals_path)
+
+    peer_review_journals_list = peer_review_journals.split('\n')
+    eq_network_journals_list = eq_network_journals.split('\n')
+
+    
+    modified_journal_query = "Is the given research paper published in any of the following journals: " + ", ".join(peer_review_journals_list) + "?"
+
+    example_journal = {"query":modified_journal_query,
+                       "Answer": "The article is published in the Lancet."}
+    example_author = {"query":author_query,
+                       "Answer": "Corresponding author. Stephanie J. Sohl, Ph.D., Department of Social Sciences & Health Policy, Wake Forest School of Medicine, Medical Center Boulevard, Winston-Salem, NC 27157, USA, [email protected]"}
+
+    formatted_journal_example = format_example(example_journal)
+    formatted_author_example = format_example(example_author)
+    
+    qa_author_prompt_with_example = info_prompt.replace("{few_shot_examples}", formatted_author_example)
+    qa_journal_prompt_with_example = info_prompt.replace("{few_shot_examples}", formatted_journal_example)
+
+    info_llm = OpenAI(model="gpt-4-1106-preview", temperature=model_temperature, max_tokens=output_token_size)
+    pdf_info_query = PDFQueryEngine(documents, info_llm, embed_model, (info_prompt))
+    info_query_engine = pdf_info_query.setup_query_engine()
+    journal_result = info_query_engine.query(modified_journal_query).response
+    author_result = info_query_engine.query(author_query).response
+    
+    
+    pdf_criteria_query = PDFQueryEngine(documents, info_llm, embed_model, (general_prompt))
+
+    # Check for prior registration
+    nlp_methods = KeywordSearch(merged_chunks)
+    eq_journal_result = nlp_methods.find_journal_name(journal_result, eq_network_journals_list)
+    peer_journal_result = nlp_methods.find_journal_name(journal_result, peer_review_journals_list)
+
+    registration_result = nlp_methods.check_registration()
+
+    # Convert your asynchronous operations into a synchronous context using asyncio.run
+    async def async_evaluation():
+        # This assumes that evaluate_with_llm_async is an async version of your method
+        return await pdf_criteria_query.evaluate_with_llm_async(registration_result, peer_journal_result, eq_journal_result, queries)
+
+        
+    # Evaluate with OpenAI model
+    total_score, criteria_met, score_percentage, reasoning = asyncio.run(async_evaluation())
+
+    reasoning_html = "<ul>"
+    for query, reason in zip(criteria, reasoning):
+        reasoning_html += f"<li style='font-size: 18px;'><strong style='color: forestgreen;'>{query}</strong> <br> Reasoning: {reason}</li>"
+    reasoning_html += "</ul>"
+
+    # Generate the score bar HTML
+    score_bar_html = generate_score_bar(total_score, n_criteria)
+
+    # Return the score as a string and the reasoning as HTML
+    return str(round((total_score / n_criteria) * 100)) + "/100", score_bar_html, reasoning_html, author_result
+
+
+with gr.Blocks(theme=gr.themes.Glass(
+    text_size="sm", 
+    font=[gr.themes.GoogleFont("Inconsolata"), "Arial", "sans-serif"], 
+    primary_hue="neutral", 
+    secondary_hue="gray")) as demo:
+    
+    gr.Markdown("## Med Library")
+    
+    with gr.Row():
+        file_upload = gr.File(label="Choose a paper", file_types=['.pdf'])
+    
+    with gr.Row():
+        models = ["Model 1", "Model 2"]
+        model_choice = gr.Dropdown(models, label="Choose a model", value="Model 1")
+        submit_button = gr.Button("Evaluate")
+
+    score_output = gr.Textbox(label="Final Score:", interactive=False)
+    score_bar_output = gr.HTML()
+    reasoning_output = gr.HTML()
+
+    # Heading for Author Information
+    gr.Markdown("## Author Information")
+
+    # Output for dynamically generated author information
+    author_info_output = gr.Markdown()
+        
+    # Set the click event for the button
+    submit_button.click(
+        fn=process_pdf,
+        inputs=[file_upload, model_choice],
+        outputs=[score_output, score_bar_output, reasoning_output, author_info_output]
+    )
+
+
+#Launch the app
+demo.launch(share=True, server_name="0.0.0.0", server_port=7860)
+
+# Main route for file upload and display results
+# @app.route('/', methods=['GET', 'POST'])
+# def upload_and_display_results():
+#     total_score = 0
+#     score_percentage = 0
+#     reasoning = []
+#     criteria_met = 0
+
+#     if request.method == 'POST':
+#         # Check if the post request has the file part
+#         if 'file' not in request.files:
+#             flash('No file part')
+#             return redirect(request.url)
+#         file = request.files['file']
+#         # If user does not select file, browser also submits an empty part without filename
+#         if file.filename == '':
+#             flash('No selected file')
+#             return redirect(request.url)
+#         if file and allowed_file(file.filename):
+#             try:
+#                 # Process the PDF file
+#                 pdf_processor = PDFProcessor_Unstructured(pdf_processing_config)
+#                 merged_chunks, tables = pdf_processor.process_pdf_file(file)
+#                 documents = [Document(text=t) for t in merged_chunks]
+
+#                 # LLM Model choice
+#                 try:
+#                     if llm_model == "gpt-4" or llm_model == "gpt-3.5-turbo":
+#                         llm = OpenAI(model=llm_model, temperature=model_temperature, max_tokens=output_token_size)
+
+#                     elif llm_model == "mistralai/Mixtral-8x7B-Instruct-v0.1":
+#                         if any(param is None for param in [model_context_window, output_token_size, model_temperature, hf_token]):
+#                             raise ValueError("All parameters are required for Mistral LLM.")
+
+#                         llm = MixtralLLM(context_window=model_context_window, num_output=output_token_size,
+#                             temperature=model_temperature, model_name=llm_model, api_key=hf_token)
+#                     else:
+#                         raise ValueError(f"Unsupported language model: {llm_model}")
+
+#                 except Exception as e:
+#                     logger.error(f"Error initializing language model '{llm_model}': {e}", exc_info=True)
+#                     raise  # Or handle the exception as needed
+
+#                 # Embedding model choice for RAG
+#                 try:
+#                     if embed == "openai":
+#                         embed_model = OpenAIEmbedding()
+
+#                     elif embed == "huggingface":
+#                         if embed_model_name is None:
+#                             # Set to default model if name not provided
+#                             embed_model_name = "BAAI/bge-small-en-v1.5"
+#                             embed_model = HuggingFaceEmbedding(embed_model_name)
+#                         else:
+#                             # Use the specified model name
+#                             embed_model = HuggingFaceEmbedding(embed_model_name)
+#                     else:
+#                         raise ValueError(f"Unsupported embedding model: {embed_model}")
+
+
+#                 except Exception as e:
+#                     logger.error(f"Error initializing embedding model: {e}", exc_info=True)
+#                     raise
+
+
+
+#                 # Prompts and Queries
+#                 utils = base_utils()
+#                 general_prompt = utils.read_from_file(general_prompt_path)
+#                 info_prompt = utils.read_from_file(info_prompt_path)
+
+#                 peer_review_journals = utils.read_from_file(peer_review_journals_path)
+#                 eq_network_journals = utils.read_from_file(eq_network_journals_path)
+
+#                 peer_review_journals_list = peer_review_journals.split('\n')
+#                 eq_network_journals_list = eq_network_journals.split('\n')
+
+
+#                 modified_journal_query = "Is the given research paper published in any of the following journals: " + ", ".join(peer_review_journals_list) + "?"
+
+#                 pdf_info_query = PDFQueryEngine(documents, llm, embed_model, (info_prompt))
+#                 info_query_engine = pdf_info_query.setup_query_engine()
+#                 journal_result = info_query_engine.query(modified_journal_query).response
+
+
+#                 pdf_criteria_query = PDFQueryEngine(documents, llm, embed_model, (general_prompt))
+
+#                 # Check for prior registration
+#                 nlp_methods = KeywordSearch(merged_chunks)
+#                 eq_journal_result = nlp_methods.find_journal_name(journal_result, eq_network_journals_list)
+#                 peer_journal_result = nlp_methods.find_journal_name(journal_result, peer_review_journals_list)
+#                 registration_result = nlp_methods.check_registration()
+
+
+#                 # Evaluate with OpenAI model
+#                 total_score, criteria_met, score_percentage, reasoning = pdf_criteria_query.evaluate_with_llm(registration_result, peer_journal_result, eq_journal_result, queries)
+
+
+#             except Exception as e:
+#                 logging.exception("An error occurred while processing the file.")
+#                 # Consider adding a user-friendly message or redirect
+#                 flash('An error occurred while processing the file.')
+#                 return redirect(request.url)
+
+#     return render_template('index.html',
+#                        total_score = total_score,
+#                        score_percentage = score_percentage,
+#                        criteria_met = criteria_met,
+#                        reasoning = reasoning)
+
+
+# if __name__ == '__main__':
+#     app.run(debug=True)

data/data1/git.keep

@@ -0,0 +1 @@
+

data/git.keep

@@ -0,0 +1 @@
+

data/prompts/eq_network_journals.txt

@@ -0,0 +1,9 @@
+The New England Journal of Medicine (NEJM)
+The Lancet
+Journal of the American Medical Association (JAMA)
+British Medical Journal (BMJ)
+Annals of Internal Medicine
+Nature Medicine
+Journal of Clinical Oncology
+Journal of Clinical Investigation
+Pediatrics

data/prompts/peer_review_journals.txt

@@ -0,0 +1,17 @@
+The New England Journal of Medicine (NEJM)
+The Lancet
+Journal of the American Medical Association (JAMA)
+British Medical Journal (BMJ)
+Annals of Internal Medicine
+Nature Medicine
+Journal of Clinical Oncology
+Journal of Clinical Investigation
+Pediatrics
+Cell
+Journal of Experimental Medicine
+Circulation
+Science Translational Medicine
+Archives of Internal Medicine
+Journal of Immunology
+Proceedings of the National Academy of Sciences (PNAS)
+Anesthesia & Analgesia

data/prompts/prompt_gpt.txt

@@ -0,0 +1,19 @@
+"""
+Provided below is context information essential for the evaluation of a medical research paper.
+
+-------------------------------------
+{context_str}
+-------------------------------------
+
+As an expert in medical research and an experienced peer reviewer, your task is to thoroughly evaluate the provided medical research paper. The evaluation should be specifically tailored to the context information given above. 
+Your assessment will be guided by the specific query: {query_str}. You are required to assign a binary score of either 0 or 1. This score should be a direct reflection of the quality, relevance, and accuracy of the research paper in relation to the query.
+In addition to the score, it is crucial that you provide concise and clear reasoning for your evaluation. Your justification should directly reference specific sections or findings within the paper, demonstrating how they influenced your scoring decision.
+Keep your reasoning consice without missing any critical information.
+
+In cases where a query cannot be satisfactorily answered due to a lack of information or clarity in the research paper, please indicate this explicitly in your response. Such instances should be factored into your scoring, reflecting the incompleteness or inadequacy of the paper in addressing the query at hand.
+
+Please adhere to the following format when presenting your answers:
+Score: [Insert score here, either 0 or 1]
+Reasoning: [Provide a brief and clear justification for your score, citing specific parts of the paper]
+This structured approach ensures a comprehensive and fair assessment of the research paper, based on the specific aspects highlighted in the queries.
+"""

data/prompts/prompt_info.txt

@@ -0,0 +1,9 @@
+"""
+Context information is provided below.
+---------------------
+{context_str}
+---------------------
+As an information extraction specialist and based on the context information, answer the query: {query_str}. Focus on brevity and clarity. Your aim is to extract key information and provide direct, to-the-point answers to specific questions about a research article. Answers should be concise, avoiding unnecessary details.
+Format the answer in a readable way, omitting any irrelevant special characters.
+In cases where a query cannot be adequately answered due to insufficient or unclear information in the article, state this explicitly.
+"""

data/prompts/prompt_mistral.txt

@@ -0,0 +1,12 @@
+"Context information is below.\n"
+"---------------------\n"
+"{context_str}\n"
+"---------------------\n"
+"As a medical research expert and peer reviewer, evaluate the medical research paper based on the relevant sections provided as context information.\n" 
+"Your evaluation should be based on the specific query: {query_str} \n"
+"For each query, provide a binary score of either 0 or 1. \n"
+"Justify your score with concise reasoning, citing specific parts of the paper.\n"
+"If a query cannot be answered due to insufficient information, state this clearly and score accordingly.\n"
+"Answer format is given below. \n"
+"Score: \n"
+"Reasoning: "

few_shot.json

@@ -0,0 +1,47 @@
+{
+    "1": {
+        "query": "Is anything in the article shared (data, code)? Look for the words like Supporting, Additional, supplementary  information/code/material/datashar -ing/-ed/-e, available, reproducibility and similar  + links, appendix",
+        "score": 1,
+        "reasoning": "Supplementary materials\n Supplementary material associated with this article can be found in the online version at doi:10.1016/j. ebiom.2022.103945.\n \n Data sharing\n All relevant data are within the paper and its supplementary files. The raw data used and/or analysed during the study are available in the Genome Sequence Archive for Human repository [HRA001933 in https://bigd.big.ac.cn/gsa-human/]."
+    },
+    "2": {
+        "query": "Has anything in the article been registered (in advance)?",
+        "score": 1,
+        "reasoning": "This study was registered and the inclusion criteria for patients were presented on ClinicalTrials.gov with the number NCT02533271, STELLAR. The primary endpoint was 3-year relapse-free survival, defined as the time from the date of randomization to the first occurrence of local-regional failure or distant metastasis. The secondary objectives were 3-year local relapse-free survival, distant metastasis-free survival, and overall survival."
+    },
+    "3": {
+        "query": "Does the article follow any reporting guidelines? To answer this question, follow the 3 steps sequentially. If any of the steps is true, assign a score of 1 and if all the steps are false, give a score of 0. STEP 1. Look for ISRCTN registry. STEP 2. Look if it is published in either The Lancet, The New England Journal of Medicine (NEJM), Journal of the American Medical Association (JAMA), British Medical Journal (BMJ), Annals of Internal Medicine, Nature Medicine, or Journal of Clinical Oncology. STEP 3. Look for one of the following guidelines, CONSORT for randomized controlled trials, PRISMA for meta\u2010analyses or systematic reviews, MOOSE for Meta-analyses of observational studies, STARD for diagnostic/prognostic studies, ARRIVE for animal pre-clinical studies, STROBE observational studies,SPIRIT for study protocols, CARE for case reports, AGREE for clinical practice guidelines, SRQR for qualitative researches,SQUIRE for quality improvement studies, SPIRIT Statement: Standard Protocol Items: Recommendations for Interventional Trials, PRIMER Collaboration: PRESENTATION AND INTERPRETATION OF MEDICAL RESEARCH, MIBBI: Minimum Information for Biological and Biomedical Investigations, COREQ: Consolidated Criteria for Reporting Qualitative Research, MDAR (Materials Design Analysis Reporting) reproducibility checklist is not a traditional reporting guideline like CONSORT or PRISMA. Instead, it's a tool designed to enhance the reproducibility and transparency of scientific research, REMARK (Reporting Recommendations for Tumor Marker Prognostic Studies).",
+        "score": 1,
+        "reasoning": "The Lancet"
+    },
+    "4": {
+        "query": "Is the methodology described in detail (where, when, how, what, who)?",
+        "score": 1,
+        "reasoning": "Methods Sixty patients with LARC from a multicentre, phase II/III randomized trial were included, with tissue and blood samples collected. For each cfDNA sample, we profiled MRD using 3 approaches: personalized assay targeting tumour-informed mutations, universal panel of genes frequently mutated in colorectal cancer (CRC), and low depth sequencing for copy number alterations (CNAs).\n \n Patients enrolled were randomly assigned in a 1:1 ratio\n to short-course preoperative radiotherapy (SCPRT, 5 Gy\n x 5 alone) with neoadjuvant chemotherapy (NCT) (4\n cycles of capecitabine plus oxaliplatin regimen) and preoperative long-course chemoradiotherapy (2 Gy x 25\n with capecitabine). The treatment strategies in these\n two groups were described in detail in STELLAR registration file. \n \n For each patient, we selected up to 22 somatic mutations from the tumour tissue. We designed customized\n primers targeting the mutations and used the primers to profile the matched cfDNA with Mutation Capsule\n technology as previously described. Briefly, the cfDNA was ligated to a customized adaptor and amplified to\n produce a whole genome library that was subsequently used as a template and amplified with customized primers. Multiplex PCR primer pairs for the two rounds of nested amplification were designed using Oligo software (v7.53) and their uniqueness were verified in the human genome (http://genome.ucsc.edu/) to ensure amplification efficiency. In the first round of amplification, the whole genome library was amplified in 9 cycles of PCR using a target-specific primer and a primer matching the adapter sequence. A second round of 14 cycles of amplification was performed with one pair of nested primers matching the adapter and the target region to further enrich the target region and add the Illumina adapter sequences to the construct. The final libraries were sequenced using the Illumina NovaSeq 6000 platform at a median depth of 6835\u00a3 after removing duplicate molecules. The median on-target ratio of reads mapped to the target region was 80%. The clean reads were mapped to the human reference hg19 genome using 'BWA (v0.7.15) mem' with the default parameters. Samtools mpileup was used to identify somatic mutations, including SNVs and INDELs, across the targeted regions of interest. Each uniquely labelled template was amplified, resulting in a certain number of daughter molecules with the same sequence (defined as a UID family). If a mutation is pre-existing in the template molecule (original cfDNA) used for amplification, the mutation should be present in each daughter molecule containing the UID (barring any subsequent replication or sequencing errors). A UID family in which at least 80% of the family members have the same mutation is called the EUID family, indicating that it harbours a mutation that should be true instead of a false-positive mutation due to amplification or sequencing error. The mutant allelic fraction was calculated by dividing the number of alternative EUID families by the sum of alternative and reference families. Tissue-specific mutations with at least one distinct paired duplex EUID family or four distinct EUID families were subsequently manually checked in IGV and verified using a cross-validation method. The candidate mutations were annotated with Ensemble Variant Effect Predictor (VEP)."
+    },
+    "5": {
+        "query": "Is the data collection processes described in detail (where, when, how, what, who)?",
+        "score": 1,
+        "reasoning": "The tumour tissues were collected at the diagnostic stage by biopsy sampling, and peripheral blood was collected in EDTA Vacutainer tubes (BD Diagnostics; Franklin Lakes, NJ, USA) and centrifuged within 2 h of collection at 4000 \u00a3 g for 10 min to separate plasma and blood cells. Plasma was centrifuged a second time at 12,000 \u00a3 g for 10 min at 4\u00b0C to remove any remaining cellular debris and stored at -80\u00b0C.\n \n Clinical serum levels of the biomarkers carcinoembryonic antigen (CEA) and carbohydrate antigen 19-9 (CA 19-9) were monitored at baseline, before surgery and after surgery. CEA and CA19-9 levels were measured with immunoelectrochemiluminescence, with CEA concentrations of < 5.0 ng/mL and CA19-9 concentrations of < 27.0 U/mL considered within the reference range. Chest/abdominal/pelvic CT scans were performed every 3 months during the first two years and then every 6 months for a total of 5 years. Clinicians were blinded to the ctDNA results during the courses of neoadjuvant therapy.\n \n Genomic DNA (gDNA) was extracted from fresh frozen tumour biopsies and WBCs with the QIAamp DNA Mini Kit (Qiagen; Germantown, MD, USA), and cfDNA was extracted from 1.5-4.5 mL of plasma with the Apostle MiniMax cfDNA isolation kit (C40605, Apostle; San Jose, CA, USA). Targeted sequencing of a panel of 509 genes or exomes was performed using genomic DNA obtained from tumour tissue and WBCs as previously described.\n \n Briefly, the raw data (FASTQ file) were aligned to the UCSC human reference genome hg19 using BurrowsWheeler aligner software (BWA, v0.7.15). Basic processing, marking duplicates, local realignments and score recalibration were performed using The Genome Analysis Toolkit (GATK, v3.6), Picard (v2.7.1) and Samtools (v1.3.1). Candidate somatic mutations were detected by comparing sequencing data from tumour tissue samples with MuTect1 and Strelka. All selected mutations were further validated by performing a manual inspection using Integrated Genome Viewer (IGV).\n \n The raw sequencing data were treated as described above, and the next segmentation analysis was performed using QDNASeq (v1.14.0). The resulting output files were summarized using R software (v4.0.3). Overlap analysis was performed using bedtools (v2.17.0) and plotted with UpSetR (v1.4.0) within the R package (v4.0.3). Chromosome arm-level alterations show cancer-specific patterns. For example, a hierarchical clustering analysis of mean arm-level calls performed across 3,000 TCGA samples revealed that gastrointestinal tumours clustered with gains of chromosomes 8q, 13q, and 20. Some of these CNAs, including gains of chromosomes 1q, 8q, 7,12q, 13q, and 20q and loss of chromosomes 1p, 20p, and 22q, were also recurrently identified in our cohort as hot CNAs (34 baseline plasma samples from patients with LARC compared with 70 plasma samples from healthy controls). Therefore, we defined the CNA number as the sum of hot chromosome arms altered (|Z| > 2) to represent the level of copy number variation."
+    },
+    "6": {
+        "query": "Is there any sample description? eg. size, demographics, recruitment, in-/exclusion criteria",
+        "score": 1,
+        "reasoning": "Patient characteristics and tissue mutation identification\n Patients with locally advanced rectal cancer (n = 82; cT3- 4N0 or cTanyN1-2) were enrolled in the trial from December 30, 2016, to October 8, 2018. Twenty-two patients were excluded due to the lack of plasma samples obtained after NAT (Figure 1a). Thirty-one patients were treated with long-course neoadjuvant chemoradiotherapy (LCRT), and 29 patients were treated with short-course neoadjuvant radiotherapy (SCPRT) with neoadjuvant chemotherapy (Table 1). The median follow-up period was 33.25 months (range, 9.6342.43 months). Seventeen (28.33%) patients were diagnosed with local relapse or metastasis during follow-up, including 5/17 (29.41%) with local relapse, 6/17 (35.29%) with liver metastasis and 6/17 (35.29%) with lung metastasis (Table S1).\n One hundred ninety-six blood samples were available during the treatment process, including baseline (collected before NAT, n = 42), in-process (collected during NAT, n = 35), post-NAT (collected 2 weeks after SCPRT or LCRT, n = 60) and pre-TME (collected before surgery, n = 59) samples (Figure 1a). We performed targeted sequencing with a panel of 509 genes or exome sequencing on the genomic DNA isolated from the tumour tissue and matched WBCs, and then identified a median of 51 (range, 3-177) somatic mutations in each tumour (Table S2). The mutational landscape of the top 15 most significantly mutated genes in the cohort was shown in Figure 1b. Customized primers were designed to profile up to 22 somatic mutations in the matched cfDNA with Mutation Capsule technology (Table S3) as previously described.\n \n Thirty-five patients with a positive ctDNA fraction at baseline were analysed (35/42 patients) to explore the performance of the ctDNA fraction in monitoring the NAT response. With ctDNA clearance defined as ratio of post-NAT ctDNA fraction to baseline ctDNA fraction below 2% (median value of the ratio), 19 (54.29%) patients showed no clearance at the post-NAT time point relative to baseline ctDNA fraction values (Figures 5, S3b). For patients with or without ctDNA clearance, there were 9/16 (56.25%) and 18/19 (94.74%) exhibited nonpCR/cCR (clinical complete response), respectively."
+    },
+    "7": {
+        "query": "Does the article describe the data analysis process?",
+        "score": 1,
+        "reasoning": "Statistics\n In this clinical cohort-based investigative study, the primary aim was to test the hypothesis that changes in the ctDNA fraction during treatment dynamically reflect minimal residual disease. Correlation analysis between input and estimated ctDNA in ctDNA fraction model and analysis of variance for the assessment of longitudinal plasma samples were the exploratory studies. Method for hypothesis testing and survival analysis was commonly used by previous researchers. Specifically, correlation analysis used Spearman\u2019s correlation analysis. For continuous variables, differences in ctDNA fractions between recurrence and non-recurrence groups were assessed with MannWhitney (rank sum) test, ctDNA fractions across treatment courses of NAT were assessed by Kruskal-Wallis test and post hoc using Dunn's multiple comparisons test, and the ctDNA fraction was assessed for patients with paired baseline and post-NAT data using Wilcoxon matched-pairs signed rank test. Differences in clinical characteristics between patients with positive and negative ctDNA fractions were evaluated with Fisher\u2019s exact test for categorical variables. These statistical analyses were performed with Prism 8 software (v8.4.3). Relapse-free survival (RFS) was measured from the date of randomization to the first occurrence of local-regional failure or distant metastasis. The univariate analysis was conducted using the KaplanMeier method with the log-rank test. HR values were calculated using univariate Cox proportional hazard models. The multivariate analysis was based on the Cox proportional hazard model in which the common important factors, such as age, sex, and clinical risk (according to the ESMO guidelines) were included. The survival model was evaluated with the C-index. The KaplanMeier curves were verified by performing a time-dependent receiver operating characteristic (ROC) curve analysis, and the area under the curve (AUC) was calculated to evaluate the prognostic performance. These analyses were performed using R software (v4.0.3). P values < 0.05 from a 2-sided test were considered statistically significant in all analyses. A sample of fifty patients was needed to achieve the power of 0.8 in this study as previously described.\n \n We next checked longitudinal status of the ctDNA fraction and its possible association with the disease course, therapeutic effect and survival status of all 60 patients (Figure 4a). Compared with baseline and in-process samples, a clear trend of a reduced post-NAT ctDNA fraction was observed in both the recurrence and nonrecurrence groups (Figure 4b), which highlighted the significant therapeutic effect of NAT. We noticed a more substantial reduction in the ctDNA fraction during baseline, in-process and post-NAT stages within the nonrecurrence group (Dunn\u2019s multiple comparison test, baseline vs. in-process: P = 0.0130; baseline vs. postNAT: P < 0.0001; in-process vs. post-NAT: P = 0.0009) compared to the recurrence group (Dunn\u2019s multiple comparison test, baseline vs. in-process: P > 0.9999; baseline vs. post-NAT: P = 0.1819; in-process vs. post-NAT: P = 0.4114) (KruskalWallis test, nonrecurrence group, P < 0.0001; recurrence group, P = 0.113) (Figure 4b). Moreover, the post-NAT ctDNA fraction status exhibited the strongest association with RFS, followed by the status at the in-process (HR = 3.61; 95% CI, 0.73-17.91; log-rank P = 0.093) and baseline stages (HR = 1.58; 95% CI, 0.20-12.67; log-rank P = 0.66). For the 17 patients experiencing recurrence, the median lead time between the detection of positive post-NAT ctDNA fraction and finding of radiological recurrence was 10.2 months (range, 0.1-33.2 months) (Wilcoxon matched-pairs signed rank test, P = 0.0001) (Figure S3a). We explored whether ctDNA fraction dynamics were linked to RFS by specifically focusing on the 42 patients with both baseline and post-NAT samples and observed a decreased ctDNA fraction in most patients (85.71%, 36/42). For the 9 patients experiencing recurrence, the ctDNA fraction after NAT increased in 4 (44.44%) patients and decreased but was still positive in 4 (44.44%) patients. In the nonrecurrence group (n = 33), the ctDNA fraction decreased to undetectable levels in 30 patients (90.90%) (Figure 4c). These data showed better predictive value of the post-NAT ctDNA fraction status than the ctDNA fraction dynamics (HR = 7.40; 95% CI: 1.97-27.82; log-rank P = 0.00053; sensitivity of 44.44% and specificity of 93.94%) for RFS estimation. The ctDNA fraction (post-NAT) in MRD-positive samples varied significantly from 0.05% to 12.74%. We divided the post-NAT samples into two groups to test if the ctDNA fraction values were correlated with the recurrence status: highly positive ctDNA fraction ( 1%) and moderately positive ctDNA fraction (0.05%-1%). The RFS of the 3 patients with highly positive post-NAT ctDNA fractions was shorter (< 200 days) than that of the moderately positive group (Figure 4d). In patient FL126 with two post-NAT plasma samples, the ctDNA fraction in plasma was moderately positive (0.16%) at 20 days after NAT and highly positive (3.50%) at 141 days, and lung metastases appeared in this patient only 43 days after the second time point (Figure 4e). In patient FL199 with a moderately positive ctDNA fraction (0.23%), local relapse occurred 306 days later (Figure 4e). The dynamic ctDNA fraction in the remaining samples was shown in Figure S4.\n \n The association between ctDNA fraction clearance and response to neoadjuvant therapy was significant (Fisher's exact test, P = 0.013)."
+    },
+    "8": {
+        "query": "Were measures taken to avoid or minimize systematic bias?",
+        "score": 1,
+        "reasoning": "This study had several limitations. First, the sample size was modest, and a limited number of patients were included in each subgroup, such as longitudinal plasma samples or patients who accepted LCRT/SCPRT. Second, intervention studies are required to explore the potential clinical utility of ctDNA to guide therapeutic decision-making and to determine whether the administration of neoadjuvant chemotherapy under ctDNA guidance may exert a positive effect on survival.\n \n Declaration of interests\n YCJ is one of the cofounders, has owner interest in Genetron Holdings, and receives royalties from Genetron. The other authors have declared that no competing interest exists.\n \n Role of the funding source\n The sponsors did not have any role in the study design, data collection, data analyses, interpretation, or writing of the manuscript.\n \n Funding\n The National Key R&D Program of China, Beijing Municipal Science & Technology Commission, National Natural Science Foundation of China, and CAMS Innovation Fund for Medical Sciences.\n \n Acknowledgements\n The authors would like to thank Ying Zhang for the assistance with sample collection and Pei Wang for the primary technical assistance. This work was supported by financial support were as follows: the National Key R&D Program of China [2021YFC2500900], Beijing Municipal Science & Technology Commission [Z181100001718136], National Natural Science Foundation of China [82073352], and CAMS Innovation Fund for Medical Sciences [2017-I2M-1-006 and 2021-I2M-1- 067]. The sponsors had no role in study design, data collection, data analyses, interpretation, and writing of the manuscript.\n \n Funding The Beijing Municipal Science & Technology Commission, National Natural Science Foundation of China, and CAMS Innovation Fund for Medical Sciences"
+    },
+    "9": {
+        "query": "Has the article been published in a journal?",
+        "score": 1,
+        "reasoning": "the lancet"
+    }
+}

model_config.json

@@ -0,0 +1,46 @@
+{
+    "pdf_processing": {
+        "extract_images": false,
+        "infer_table_structure": true,
+        "strategy": "fast",
+        "chunking_strategy": "by_title",
+        "model_name": "yolox",
+        "max_characters": 10000,
+        "combine_text_under_n_chars": 100
+      },
+    "allowed_extensions": "pdf",
+    "embeddings": "huggingface",
+    "embeddings_model": "BAAI/bge-small-en-v1.5",
+    "llm_model": "gpt-4",
+    "model_temp": 0.2,
+    "max_tokens": 512,
+    "context_window": 5000,
+    "UPLOAD_FOLDER": "../path/to/upload/folder",
+    "GPT_PROMPT_PATH": "data/prompts/prompt_gpt.txt",
+    "MISTRAL_PROMPT_PATH": "data/prompts/prompt_mistral.txt",
+    "INFO_PROMPT_PATH": "data/prompts/prompt_info.txt",
+    "peer_review_journals_path": "data/prompts/peer_review_journals.txt",
+    "eq_network_journals_path": "data/prompts/eq_network_journals.txt",
+    "queries": [
+        "Is anything in the article shared (data, code)? Look for the words like Supporting, Additional, supplementary  information/code/material/datashar -ing/-ed/-e, available, reproducibility and similar  + links, appendix",
+        "Has anything in the article been registered (in advance)?",
+        "Does the article follow any reporting guidelines? To answer this question, follow the 3 steps sequentially. If any of the steps is true, assign a score of 1 and if all the steps are false, give a score of 0. STEP 1. Look for ISRCTN registry. STEP 2. Look if it is published in either The Lancet, The New England Journal of Medicine (NEJM), Journal of the American Medical Association (JAMA), British Medical Journal (BMJ), Annals of Internal Medicine, Nature Medicine, or Journal of Clinical Oncology. STEP 3. Look for one of the following guidelines, CONSORT for randomized controlled trials, PRISMA for meta‐analyses or systematic reviews, MOOSE for Meta-analyses of observational studies, STARD for diagnostic/prognostic studies, ARRIVE for animal pre-clinical studies, STROBE observational studies,SPIRIT for study protocols, CARE for case reports, AGREE for clinical practice guidelines, SRQR for qualitative researches,SQUIRE for quality improvement studies, SPIRIT Statement: Standard Protocol Items: Recommendations for Interventional Trials, PRIMER Collaboration: PRESENTATION AND INTERPRETATION OF MEDICAL RESEARCH, MIBBI: Minimum Information for Biological and Biomedical Investigations, COREQ: Consolidated Criteria for Reporting Qualitative Research, MDAR (Materials Design Analysis Reporting) reproducibility checklist is not a traditional reporting guideline like CONSORT or PRISMA. Instead, it's a tool designed to enhance the reproducibility and transparency of scientific research, REMARK (Reporting Recommendations for Tumor Marker Prognostic Studies).",
+        "Is the methodology described in detail (where, when, how, what, who)?",
+        "Is the data collection processes described in detail (where, when, how, what, who)?",
+        "Is there any sample description? eg. size, demographics, recruitment, in-/exclusion criteria",
+        "Does the article describe the data analysis process?",
+        "Were measures taken to avoid or minimize systematic bias?",
+        "Has the article been published in a journal?"],
+    "criteria": [
+        "Data and code sharing.",
+        "Has anything in the article been registered (in advance)?",
+        "Does the article follow any reporting guidelines?",
+        "Description of methodology",
+        "Data collection processes",
+        "Sample description. eg. size, demographics, recruitment, in-/exclusion criteria",
+        "Data analysis process",
+        "Measures to minimize systematic bias",
+        "Peer Review"],
+    "journal_query": "Is the given research paper published in any of the following journals: {}?",
+    "author_query": "Give me details about the institutions (like university or hospital) and contact details (eg. email) of the corresponding author."    
+}

requirements.txt

@@ -0,0 +1,39 @@
+shiny
+beautifulsoup4
+chromadb
+gradio
+cohere
+faiss-cpu
+Flask
+langchain
+langchainhub
+llama-index == 0.9.35
+llmsherpa
+lxml
+onnxruntime
+unstructured
+bs4
+evaluate
+faiss-cpu
+numpy
+openai
+Pillow
+PyPDF2
+pydantic
+rank-bm25
+requests
+rapidocr-onnxruntime
+rouge-score
+scikit-learn
+sentence-transformers
+tiktoken
+transformers
+tesseract
+pdf2image
+pdfminer.six
+opencv-python
+pikepdf
+pypdf
+qdrant-client
+unstructured-inference
+unstructured-pytesseract

templates/index.html

@@ -0,0 +1,183 @@
+<!doctype html>
+<html>
+<head>
+  <title>Upload and Results</title>
+  <!-- Include Google Fonts -->
+  <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap" rel="stylesheet">
+  <style>
+    body {
+        font-family: 'Roboto', sans-serif;
+        background-color: #f4f4f4;
+        overflow: auto;
+        width: 100%;
+        margin: 0;
+        padding: 0;
+        display: flex;
+        flex-direction: column; /* Stack flex items vertically */
+        align-items: center; /* Center items horizontally */
+        justify-content: flex-start; /* Align items to the start of the container vertically */
+        min-height: 100vh; /* Use min-height instead of height to accommodate content taller than the viewport */
+    }
+
+    table {
+        width: 100%; /* Adjust the width as needed */
+        border-collapse: collapse; /* Collapse borders for a tighter look */
+    }
+
+    th, td {
+        border: 1px solid #ddd; /* Adjust the border size as needed */
+        text-align: left;
+        padding: 5px; /* Reduce padding to decrease cell spacing */
+        height: 30px; /* Optionally reduce the height of the cells */
+    }
+    .parent-element {
+        overflow: visible; /* Ensures content is not cut off */
+    }
+    .container {
+        background-color: white;
+        overflow: auto;
+        border-radius: 8px;
+        box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
+        padding: 40px;
+        width: 100%; /* Set width to 100% of the viewport */
+        max-width: 700px;
+    }
+    .score-bar-container {
+      position: relative;
+      margin-top: 20px; /* Space above the score bar */
+      max-width: 100%; /* Ensures the container does not exceed the parent width */
+    }
+    .score-very-good-fill {
+        background-color: #4CAF50; /* Green */
+    }
+
+    .score-good-fill {
+        background-color: #FFEB3B; /* Yellow */
+    }
+
+    .score-ok-fill {
+        background-color: #FF9800; /* Orange */
+    }
+
+    .score-bad-fill {
+        background-color: #f44336; /* Red */
+    }
+
+    .score-very-bad-fill {
+        background-color: #9E9E9E; /* Grey */
+    }
+    .score-very-good-text {
+            color: #4CAF50; /* Green */
+      }
+
+    .score-good-text {
+            color: #FFEB3B; /* Yellow */
+      }
+
+    .score-ok-text {
+            color: #FF9800; /* Orange */
+      }
+
+    .score-bad-text {
+            color: #f44336; /* Red */
+      }
+
+    .score-very-bad-text {
+            color: #9E9E9E; /* Grey */
+      }
+
+    .score-bar {
+      background-color: #ddd;
+      border-radius: 10px;
+      height: 20px;
+      width: 100%; /* Adjusted to take the full width */
+      display: inline-block; /* Allows the score text to sit next to the score bar */
+      vertical-align: middle; /* Aligns score bar and text vertically */
+    }
+
+    .score-fill {
+      height: 100%;
+      border-radius: 10px 0 0 10px; /* Rounded corners on the left side */
+      display: inline-block;
+      vertical-align: middle;
+    }
+
+    .score-text {
+      display: inline-block;
+      vertical-align: middle; /* Align with the score bar */
+      font-weight: bold; /* Make the score text bold */
+      margin-left: 10px; /* Space between the score bar and score text */
+    }
+
+    .score-title {
+      font-size: 20px;
+      font-weight: bold;
+      margin: 20px 0;
+      color: #333;
+    }
+    .major-issues {
+      text-align: left; /* Aligns the major issues to the left */
+      padding-left: 20px; /* Padding for the bullet list */
+      list-style: inside disc; /* Bullet style */
+    }
+    form {
+      margin-bottom: 20px;
+    }
+    input[type="file"] {
+      margin-bottom: 10px;
+    }
+    input[type="submit"] {
+      cursor: pointer;
+      margin-top: 10px;
+      padding: 10px 20px;
+      border: none;
+      background-color: #4CAF50;
+      color: white;
+      border-radius: 5px;
+      font-size: 16px;
+      font-weight: bold;
+    }
+    input[type="submit"]:hover {
+      background-color: #45a049;
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <h2>Upload PDF and View Results</h2>
+
+    <!-- Upload Form -->
+    <form method="post" enctype="multipart/form-data">
+      <input type="file" name="file" required>
+      <input type="submit" value="Upload">
+    </form>
+
+    <!-- Results Section -->
+    {% if total_score is not none %}
+      <!-- GPT-4 Score Bar -->
+      <div class="score-title">Score for GPT-4:</div>
+      <div class="score-bar-container">
+        <div class="score-bar">
+          <div class="score-fill {{
+            'score-very-good-fill' if criteria_met == 9 else
+            'score-good-fill' if criteria_met >= 7 else
+            'score-ok-fill' if criteria_met >= 5 else
+            'score-bad-fill' if criteria_met >= 3 else
+            'score-very-bad-fill' }}" style="width: {{ score_percentage_gpt4 }}%;"></div>
+        </div>
+        <div class="score-text">{{ total_score }}/9</div>
+      </div>
+
+      <!-- Reasoning for GPT-4 -->
+      <h3>Reasoning:</h3>
+      <ul class="major-issues">
+        {% for issue in reasoning %}
+          <li>{{ issue }}</li>
+        {% endfor %}
+      </ul>
+      <h3>Contact Details:</h3>
+        <p> {{result}}</p>
+    {% endif %}
+  </div>
+</body>
+</html>