from haystack.nodes import TransformersDocumentClassifier from haystack.schema import Document from typing import List, Tuple import configparser import logging from pandas import DataFrame, Series from utils.preprocessing import processingpipeline try: import streamlit as st except ImportError: logging.info("Streamlit not installed") config = configparser.ConfigParser() try: config.read_file(open('paramconfig.cfg')) except Exception: logging.info("paramconfig file not found") st.info("Please place the paramconfig file in the same directory as app.py") @st.cache(allow_output_mutation=True) def load_sdgClassifier(): """ loads the document classifier using haystack, where the name/path of model in HF-hub as string is used to fetch the model object. 1. https://docs.haystack.deepset.ai/reference/document-classifier-api 2. https://docs.haystack.deepset.ai/docs/document_classifier Return: document classifier model """ logging.info("Loading classifier") doc_classifier_model = config.get('sdg','MODEL') doc_classifier = TransformersDocumentClassifier( model_name_or_path=doc_classifier_model, task="text-classification") return doc_classifier @st.cache(allow_output_mutation=True) def sdg_classification(haystackdoc:List[Document])->Tuple[DataFrame,Series]: """ Text-Classification on the list of texts provided. Classifier provides the most appropriate label for each text. these labels are in terms of if text belongs to which particular Sustainable Devleopment Goal (SDG). Params --------- haystackdoc: List of haystack Documents. The output of Preprocessing Pipeline contains the list of paragraphs in different format,here the list of Haystack Documents is used. Returns ---------- df: Dataframe with two columns['SDG:int', 'text'] x: Series object with the unique SDG covered in the document uploaded and the number of times it is covered/discussed/count_of_paragraphs. """ logging.info("Working on SDG Classification") threshold = float(config.get('sdg','THRESHOLD')) classifier = load_sdgClassifier() results = classifier.predict(haystackdoc) labels_= [(l.meta['classification']['label'], l.meta['classification']['score'],l.content,) for l in results] df = DataFrame(labels_, columns=["SDG","Relevancy","text"]) df = df.sort_values(by="Relevancy", ascending=False).reset_index(drop=True) df.index += 1 df =df[df['Relevancy']>threshold] x = df['SDG'].value_counts() df= df.drop(['Relevancy'], axis = 1) return df, x def runSDGPreprocessingPipeline(file_path, file_name)->List[Document]: """ creates the pipeline and runs the preprocessing pipeline, the params for pipeline are fetched from paramconfig Params ------------ file_name: filename, in case of streamlit application use st.session_state['filename'] file_path: filepath, in case of streamlit application use st.session_state['filepath'] Return -------------- List[Document]: When preprocessing pipeline is run, the output dictionary has four objects. For the Haysatck implementation of SDG classification we, need to use the List of Haystack Document, which can be fetched by key = 'documents' on output. """ sdg_processing_pipeline = processingpipeline() split_by = config.get('sdg','SPLIT_BY') split_length = int(config.get('sdg','SPLIT_LENGTH')) split_overlap = int(config.get('sdg','SPLIT_OVERLAP')) remove_punc = bool(int(config.get('sdg','REMOVE_PUNC'))) output_sdg_pre = sdg_processing_pipeline.run(file_paths = file_path, params= {"FileConverter": {"file_path": file_path, \ "file_name": file_name}, "UdfPreProcessor": {"removePunc": remove_punc, \ "split_by": split_by, \ "split_length":split_length,\ "split_overlap": split_overlap}}) return output_sdg_pre