<jupyter_start><jupyter_text>Disease Prediction from SymptomsDataset Source: Raw data from (http://people.dbmi.columbia.edu/~friedma/Projects/DiseaseSymptomKB/index.html)<jupyter_code># Import Dependencies
import csv
import pandas as pd
import numpy as np
from collections import defaultdict
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
df = pd.read_excel('raw_data.xlsx')

df.head()
data = df.fillna(method='ffill')
data.head()
# Process Disease and Symptom Names
def process_data(data):
    data_list = []
    data_name = data.replace('^','_').split('_')
    n = 1
    for names in data_name:
        if (n % 2 == 0):
            data_list.append(names)
        n += 1
    return data_list
disease_list = []
disease_symptom_dict = defaultdict(list)
disease_symptom_count = {}
count = 0
common_symptoms = {}
for idx, row in data.iterrows():
    
    # Get the Disease Names
    if (row['Disease'] !="\xc2\xa0") and (row['Disease'] != ""):
        disease = row['Disease']
        disease_list = process_data(data=disease)
        count = row['Count of Disease Occurrence']

    # Get the Symptoms Corresponding to Diseases
    if (row['Symptom'] !="\xc2\xa0") and (row['Symptom'] != ""):
        symptom = row['Symptom']
        symptom_list = process_data(data=symptom)
        for d in disease_list:
            for s in symptom_list:
                common_symptoms[s] = common_symptoms.get(s,0)+1
                disease_symptom_dict[d].append(s)
            disease_symptom_count[d] = count
# See that the data is Processed Correctly
disease_symptom_dict
# Count of Disease Occurence w.r.t each Disease
disease_symptom_count
symptom_list = list(common_symptoms.items())
symptom_list = sorted(symptom_list, key = lambda x: x[1], reverse=True)
with open("symptoms.txt", "w") as file:
    for tup in symptom_list:
        file.write(tup[0]+"\n")
df1 = pd.DataFrame(list(disease_symptom_dict.items()), columns=['Disease','Symptom'])
df1.head()
df1['Disease_count'] = df1['Disease'].apply(lambda x: disease_symptom_count[x])
df1.head()
def find_about_disease(disease_name):
    return list(df1['Symptom'][df1['Disease']==disease_name.lower()])
find_about_disease('diABetes')

def find_disease_from_symptoms(symptoms):
    score=0
    index=0
    for idx, items in df1.iterrows():
        curr = len(set(symptoms).intersection(set(items['Symptom'])))
        if curr > score:
            score = curr
            index=idx
    
    return df1.iloc[index]['Disease']
        
        
find_disease_from_symptoms(['suicidal, pain chest'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12875867386276021, 'fraction_numerical': 0.009252120277563608, 'mean_word_length': 3.7354014598540144, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '350085', 'license': 'MIT', 'n_tokens_mistral': 954, 'n_tokens_neox': 910, 'n_words': 195, 'path': 'Physical_Symptom_based_Disease_Prediction/Disease-Prediction-from-Symptoms.ipynb', 'repo_name': 'lancelot1337/ostProject', 'size': '2594'}