removing redundant examples

app.py

@@ -8,7 +8,7 @@ from utils import load_rule, get_attribution, get_diseases, get_drg_link, get_ic
 from transformers import AutoTokenizer, AutoModel, set_seed, pipeline
 set_seed(42)
 model_path = 'checkpoint_0_9113.bin'
-related_tensor = torch.nn.functional.normalize(torch.load('discharge_embeddings.pt'))
+related_tensor = torch.load('discharge_embeddings.pt')
 all_summaries = pd.read_csv('all_summaries.csv')['SUMMARIES'].to_list()
 
 similarity_tokenizer = AutoTokenizer.from_pretrained('kamalkraj/BioSimCSE-BioLinkBERT-BASE')
@@ -72,13 +72,20 @@ def find_related_summaries(text):
     embedding = mean_pooling(outputs, attention_mask=inputs.attention_mask)
     embedding = torch.nn.functional.normalize(embedding)
     scores = torch.mm(related_tensor, embedding.transpose(1,0))
-    scores_indices = scores.topk(k=5, dim=0)
+    scores_indices = scores.topk(k=50, dim=0)
     indices, scores = scores_indices[-1], torch.round(100 * scores_indices[0], decimals=2)
     summaries = []
+    score_set = set()
     for summary_idx, score in zip(indices, scores):
+        score = score.item()
+        if len(summaries) == 5:
+            break
         corresp_summary = all_summaries[summary_idx]
-        summary = f'{round(score.item(),2)}% Similarity Rate for the following Discharge Summary:\n\n{corresp_summary}'
+        if score in score_set:
+            continue
+        summary = f'{round(score,2)}% Similarity Rate for the following Discharge Summary:\n\n{corresp_summary}'
         summaries.append([summary])
+        score_set.add(score)
     return summaries
 
 
@@ -208,11 +215,10 @@ def main():
 
         # input to related summaries
         with gr.Row() as row:
-            with gr.Column(scale=5) as col:
-                input_related = gr.TextArea(label="Input up to 3 Related Discharge Summary/Summaries Here", visible=False)
-            with gr.Column(scale=1) as col:
-                rmv_related_btn = gr.Button(value='Remove Related Summary', visible=False)
-                sbm_btn = gr.Button(value="Submit Related Summaries", components=[input_related], visible=False)                              
+            input_related = gr.TextArea(label="Input up to 3 Related Discharge Summary/Summaries Here", visible=False)
+        with gr.Row() as row:
+            rmv_related_btn = gr.Button(value='Remove Related Summary', visible=False)
+            sbm_btn = gr.Button(value="Submit Related Summaries", components=[input_related], visible=False)                              
         
         with gr.Row() as row:
             related = gr.Dataset(samples=[], components=[input_related], visible=False, type='index')

utils.py

@@ -317,7 +317,7 @@ def visualize_text(datarecord, drg_link, icd_annotations, diseases):
         "<th style='text-align: left'>Predicted DRG</th>"
         "<th style='text-align: left'>Word Importance</th>"
         "<th style='text-align: left'>Diseases</th>"
-        "<th style='text-align: left'>ICD Codes</th>"
+        "<th style='text-align: left'>ICD Concepts</th>"
     ]
     pred_class_html = visualization.format_classname(datarecord.pred_class)
     icd_class_html = get_icd_html(icd_annotations)