chnaging coherence model

paramconfig.cfg

@@ -36,11 +36,11 @@ TOP_N = 20
 
 [coherence]
 RETRIEVER_TOP_K = 10
-MAX_SEQ_LENGTH = 256
-RETRIEVER = all-MiniLM-L6-v2
+MAX_SEQ_LENGTH = 512
+RETRIEVER = msmarco-distilbert-dot-v5
 RETRIEVER_FORMAT = sentence_transformers
 RETRIEVER_EMB_LAYER = -1
-EMBEDDING_DIM = 384
+EMBEDDING_DIM = 768
 THRESHOLD = 0.55
 SPLIT_BY = word
 SPLIT_LENGTH = 120

utils/ndc_explorer.py

@@ -3,38 +3,67 @@ import urllib.request
 import json
 
 link = "https://klimalog.die-gdi.de/ndc/open-data/dataset.json"
-def get_document(countryCode: str):
-                with urllib.request.urlopen(link) as urlfile:
-                    data =  json.loads(urlfile.read())
-                categoriesData = {}
-                categoriesData['categories']= data['categories']
-                categoriesData['subcategories']= data['subcategories']
-                keys_sub = categoriesData['subcategories'].keys()
-                documentType= 'NDCs'
-                if documentType in data.keys():
-                    if countryCode in data[documentType].keys():
-                        get_dict = {}
-                        for key, value in data[documentType][countryCode].items():
-                            if key not in ['country_name','region_id', 'region_name']:
-                                get_dict[key] = value['classification']
-                            else:
-                                get_dict[key] = value
-                    else:
-                        return None
+def get_document(country_code: str):
+    """
+    read the country NDC data from 
+    https://klimalog.die-gdi.de/ndc/open-data/dataset.json 
+    using the country code.
+    
+    Params
+    -------
+    country_code:"""
+    with urllib.request.urlopen(link) as urlfile:
+        data =  json.loads(urlfile.read())
+    categoriesData = {}
+    categoriesData['categories']= data['categories']
+    categoriesData['subcategories']= data['subcategories']
+    keys_sub = categoriesData['subcategories'].keys()
+    documentType= 'NDCs'
+    if documentType in data.keys():
+        if country_code in data[documentType].keys():
+            get_dict = {}
+            for key, value in data[documentType][country_code].items():
+                if key not in ['country_name','region_id', 'region_name']:
+                    get_dict[key] = value['classification']
                 else:
-                    return None
+                    get_dict[key] = value
+        else:
+            return None
+    else:
+        return None
 
-                country = {}
-                for key in categoriesData['categories']:
-                    country[key]= {}
-                for key,value in categoriesData['subcategories'].items():
-                    country[value['category']][key] = get_dict[key]
-                
-                return country
+    country = {}
+    for key in categoriesData['categories']:
+        country[key]= {}
+    for key,value in categoriesData['subcategories'].items():
+        country[value['category']][key] = get_dict[key]
+    
+    return country
         
-        #   country_ndc = get_document('NDCs', countryList[option])
             
-def countrySpecificCCA(cca_sent, threshold, countryCode):
+def countrySpecificCCA(cca_sent:dict, threshold:int, countryCode:str):
+    """
+    based on the countrycode, reads the country data from
+    https://klimalog.die-gdi.de/ndc/open-data/dataset.json
+    using get_documents from utils.ndc_explorer.py
+    then based on thereshold value filters the Climate Change Adaptation
+    targets assigned by NDC explorer team to that country. Using the sentences
+    create by Data services team of GIZ for each target level, tries to find the
+    relevant passages from the document by doing the semantic search.
+
+    Params
+    -------
+    cca_sent: dictionary with key as 'target labels' and manufactured sentences 
+    reflecting the target level. Please see the docStore/ndcs/cca.txt
+
+    threshold: NDC target have many categoriees ranging from [0-5], with 0 
+    refelcting most relaxed attitude and 5 being most aggrisive towards Climate 
+    change. We select the threshold value beyond which we need to focus on.
+
+    countryCode: standard country code to allow us to fetch the country specific
+    data.
+
+    """
     temp = {}
     doc = get_document(countryCode)
     for key,value in cca_sent.items():
@@ -45,6 +74,12 @@ def countrySpecificCCA(cca_sent, threshold, countryCode):
 
                 
 def countrySpecificCCM(ccm_sent, threshold, countryCode):
+    """
+    see the documentation of countrySpecificCCA. This is same instead of 
+    this gets the data pertaining to Adaptation
+    
+    """
+
     temp = {}
     doc = get_document(countryCode)
     for key,value in ccm_sent.items():