<jupyter_start><jupyter_text>Document Search : Comorbidities Minor<jupyter_code>import sys
import json
import pandas as pd
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search, Q
from collections import Counter<jupyter_output><empty_output><jupyter_text>Entity Preparation<jupyter_code>with open("input/comorb-minor.json",'r')as fin:
    comorb = json.load(fin)
meshtree_file = "input/mtrees2021.bin"
DT = {"cerebrovascular disorders":[]}
with open(meshtree_file, "r") as ftree:
    for line in ftree:
        term_tree = line.strip().split(";")
        cur_term = term_tree[0]
        cur_tree = term_tree[1]
        
        if cur_tree[0:11] == "C14.907.253":
            #print(cur_term)
            DT["cerebrovascular disorders"].append(cur_term)
#DT['cerebrovascular disorders']
comorb.update({'cerebrovascular disorders': DT["cerebrovascular disorders"]})
#comorb<jupyter_output><empty_output><jupyter_text>Entity search<jupyter_code>with open('data/t2pmid-covid.json','r')as f2:
    t2pmid = json.load(f2)
tdf = pd.DataFrame(t2pmid)
tdf.head()
tdf.shape
covid_pmids = list(set(tdf['pmid']))
len(covid_pmids)<jupyter_output><empty_output><jupyter_text>Count Comorb Publications<jupyter_code>n2count = {}
n2pmid = {}
n2pmidset = {}
allpmids = []
for name in comorb:
    n2count[name] = 0
    n2pmid[name] = 0
    n2pmidset[name] = []
    
            
for pmid,title,abstract,mesh in  zip(tdf['pmid'],tdf['title'],tdf['abstract'],tdf['mesh']):
    
        
    for name, entities in comorb.items():
        tn = 0
        for entity in entities:
            
            n_title = title.count(entity)
            tn = tn+n_title
            
            if len(abstract)>1:
                text = abstract
                
            n_abs = text.count(entity)
            tn = tn+n_abs
            
            n_mesh = mesh.count(entity)
            tn = tn+n_mesh
            
        n2count[name] = n2count[name] + tn
        
        if tn>0:
            n2pmid[name] = n2pmid[name] + 1
            n2pmidset[name].append(pmid)
            allpmids.append(pmid)
                
with open('result/n2pmidset-comorb-minor.json','w')as f3:
    json.dump(n2pmidset,f3)
len(set(allpmids))<jupyter_output><empty_output><jupyter_text>Individual Comorb Document count<jupyter_code>Data = []
for name, count in n2pmid.items():
    Data.append({"Name":name,"Text Datasets": count})
    
df = pd.DataFrame(Data)
df.to_csv("result/comorb-count-minor.csv")<jupyter_output><empty_output><jupyter_text>Covid Symptoms and Comorb<jupyter_code>with open('result/name2pmidset-covid.json','r')as f4:
    n2pmidset_covid = json.load(f4)
Data = []
allpmid = []
for name1, pmids1 in n2pmidset_covid.items():
    data = {}
    data.update({"covid symptoms" : name1})
                
    for name2, pmids2 in n2pmidset.items():
        ints = set(pmids1).intersection(set(pmids2))
        
        for item in list(ints):
            allpmid.append(item)
            
        data.update({name2:len(set(ints))})
    
    Data.append(data)
        
len(list(set(allpmid)))
cdf = pd.DataFrame(Data)
#cdf.to_csv("covid-comorb-minor.csv")
cdf = cdf.set_index("covid symptoms")
tcdf = cdf.T
tcdf = tcdf.sort_index()
tcdf
tcdf.to_csv("result/covid-comorb-minor.csv")
cdf.sum()
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

matplotlib.rcParams.update({'font.size': 13})
plt.rcParams['xtick.top'] = plt.rcParams['xtick.labeltop'] = True

plt.figure(figsize = [15,12])

#idf = newdf.set_index("name")
sns.heatmap(tcdf, annot=True, cmap = "YlGnBu", fmt = "d")


plt.rc('xtick', labelsize=25)    # fontsize of the tick labels
plt.rc('ytick', labelsize=25)    # fontsize of the tick labels

plt.savefig("plots/heatmap_comorb_minor.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13636363636363635, 'fraction_numerical': 0.01786652653704677, 'mean_word_length': 3.2441471571906355, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '323186', 'license': 'MIT', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1418, 'n_words': 235, 'path': 'Jupyter-Notebooks/Comorbodities-minor-Search.ipynb', 'repo_name': 'CaseOLAP/covid-cvd', 'size': '3806'}