<jupyter_start><jupyter_text>First we import the file and get it's contents<jupyter_code>import csv
import pandas as pd
df = pd.read_csv("./data.csv")
df<jupyter_output><empty_output><jupyter_text>Then we make a first clean of the data<jupyter_code>df = df.drop('participant_id', axis=1)
df = df.drop('recruit_date', axis=1)
df = df.drop('postal_code', axis=1)
df = df.drop('province', axis=1)
df = df.drop('family_country', axis=1)
df = df.drop('row_school', axis=1)
df = df.drop('floor_level', axis=1)
df = df.drop('survey_type', axis=1)
df = df.drop('inclusion_criteria', axis=1)
df = df.drop('housemember_symptoms___1', axis=1)
df = df.drop('housemember_symptoms___2', axis=1)
df = df.drop('housemember_symptoms___3', axis=1)
df = df.drop('housemember_symptoms___4', axis=1)
df = df.drop('housemember_symptoms___5', axis=1)
df = df.drop('persons_home', axis=1)
df = df.drop('simptomatology_date', axis=1)
df = df.drop('date_fever', axis=1)
df = df.drop('end_fever', axis=1)
df = df.drop('other_simptomatology_lliure', axis=1)
df = df.drop('sero_type_response', axis=1)
df = df.drop('sero_method', axis=1)
df = df.drop('sero_date', axis=1)
df = df.drop('sero', axis=1)
df = df.drop('cxr', axis=1)
df = df.drop('ct', axis=1)
df = df.drop('cxr_date', axis=1)
df = df.drop('thoracic_ct_date', axis=1)
df = df.drop('pcr_performed', axis=1)
df = df.drop('pcr_date', axis=1)
df = df.drop('pcr_type', axis=1)
df = df.drop('antigenic_performed', axis=1)
df = df.drop('ag_test_mark', axis=1)
df = df.drop('antigenic_date', axis=1)
df = df.drop('antigenic_sample', axis=1)
df = df.drop('othervirus_performed', axis=1)
df = df.drop('otherviruses_date', axis=1)
df = df.drop('vrs_result', axis=1)
df = df.drop('adeno_result', axis=1)
df = df.drop('flu_a_result', axis=1)
df = df.drop('flu_b_result', axis=1)
df = df.drop('adm_date', axis=1)
df = df.drop('picu_adm', axis=1)
df = df.drop('adm_hospital', axis=1)
df = df.drop('discharge_date', axis=1)
df = df.drop('final_diagnosis_text', axis=1)
df = df.drop('coviral_binary', axis=1)
df = df.drop('coviral_type', axis=1)
df = df.drop('other_viruses_text', axis=1)
df = df.drop('bacterial_infection', axis=1)
df = df.drop('bacterial_type', axis=1)
df = df.drop('clinical_and_diagnosis_data_at_the_admission_time_complete', axis=1)
list_of_shit = ['comorbi_binary',
 'cardiopathy___1',
 'cardiopathy___2',
 'cardiopathy___3',
 'hypertension___1',
 'hypertension___2',
 'hypertension___3',
 'pulmonar_disease___1',
 'pulmonar_disease___2',
 'pulmonar_disease___3',
 'asma___1',
 'asma___2',
 'asma___3',
 'nephrology___1',
 'nephrology___2',
 'nephrology___3',
 'hepatic___1',
 'hepatic___2',
 'hepatic___3',
 'neurologic___1',
 'neurologic___2',
 'neurologic___3',
 'diabetes___1',
 'diabetes___2',
 'diabetes___3',
 'tuberculosi___1',
 'tuberculosi___2',
 'tuberculosi___3',
 'idp___1',
 'idp___2',
 'idp___3',
 'neoplasia___1',
 'neoplasia___2',
 'neoplasia___3',
 'kawasaki___1',
 'kawasaki___2',
 'kawasaki___3',
 'inflammation___1',
 'inflammation___2',
 'inflammation___3',
 'vih_others___1',
 'vih_others___2',
 'vih_others___3',
 'obesity',
 'other']

for shit in list_of_shit:
    df = df.drop(shit, axis=1)

df = df.drop('flu_date', axis=1)
df = df.drop('vaccines_binary', axis=1)
df = df.drop('comorbidities_complete', axis=1)
df = df.drop('survey_end_date', axis=1)
df = df.drop('name_initials_of_the_inter', axis=1)
df = df.drop('comments', axis=1)
df = df.drop('final_outcome_complete', axis=1)
df = df.drop('id', axis=1)
df = df.drop('bus', axis=1)
df = df.drop('sports_type', axis=1)
df = df.drop('sports', axis=1)
df = df.drop('m2', axis=1)
df = df.drop('rooms', axis=1)
df = df.drop('school_symptoms_member___1', axis=1)
df = df.drop('school_symptoms_member___2', axis=1)
df = df.drop('school_symptoms_member___4', axis=1)
df = df.drop('school_symptoms_member___5', axis=1)
df = df.drop('sero_type_response_2', axis=1)
df = df.drop('sat_hb_o2', axis=1)
df = df.drop('final_outcome', axis=1)
df = df.drop('final_classification_of_th', axis=1)
list(df.columns)
df<jupyter_output><empty_output><jupyter_text>We check the number of Nan values for each column<jupyter_code>not_null = lambda x : x.notnull()
def list_nan_values(df, function):
    check_nan = lambda x : len(df) - len(df[function(x)])
    list_nan_cols = list()
    for col in df.columns:
        list_nan_cols.append((col, check_nan(df[col])))
    return list_nan_cols
list_nan = list_nan_values(df, not_null)
list_nan<jupyter_output><empty_output><jupyter_text>We check wich vaues contain every column<jupyter_code>import numpy as np
import math
def get_population(df, column):
    rs = {}
    values = list(column.dropna().unique())
    for val in values:
        rs[val] = len(df[column == val])
    rs[np.nan] = len(column.isnull()[lambda x: x == True])
    return rs
check_values = {}
for col in df.columns:
    check_values[col] = get_population(df,df[col])
check_values<jupyter_output><empty_output><jupyter_text>We clean some other values from the dataframe<jupyter_code>df = df.drop('total_days_fever', axis=1)
df = df.drop('sat_hb_o2_value', axis=1)
check_values = {}
for col in df.columns:
    check_values[col] = get_population(df,df[col])
check_values<jupyter_output><empty_output><jupyter_text>We do a heatmap in order to see the correlation between variables, for gruping it in groups and reduce the number of columns<jupyter_code>import seaborn as sns
sns.heatmap(df.corr(), vmin=-1, vmax=1, center= 0)<jupyter_output><empty_output><jupyter_text>Impossible to see nothing so we opted to make an algorithmfor making groups of strongly correlated variables. But first, we've seen that other simptomatology 4 has a big group of correlations, so we thought thatfirst we shoud separate it's values to diferent cols.<jupyter_code>other_simptomatology_fields = ['other_simptomatology', 'other_simptomatology_3','other_simptomatology_4']
list1 = list(map(lambda x: x == 1 , df['other_simptomatology']))
list2 = list(map(lambda x: x == 1 , df['other_simptomatology_3']))
list3 = list(map(lambda x: x == 1 , df['other_simptomatology_4']))
df['dolor_toracic'] = list(map(lambda x: 1 if x else 2, 
                               map(lambda x, y, z: x or y or z, list1, list2, list3)))

list1 = list(map(lambda x: x == 2 , df['other_simptomatology']))
list2 = list(map(lambda x: x == 2 , df['other_simptomatology_3']))
list3 = list(map(lambda x: x == 2 , df['other_simptomatology_4']))
df['artralgies'] = list(map(lambda x: 1 if x else 2, 
                               map(lambda x, y, z: x or y or z, list1, list2, list3)))

list1 = list(map(lambda x: x == 3 , df['other_simptomatology']))
list2 = list(map(lambda x: x == 3 , df['other_simptomatology_3']))
list3 = list(map(lambda x: x == 3 , df['other_simptomatology_4']))
df['artritis'] = list(map(lambda x: 1 if x else 2, 
                               map(lambda x, y, z: x or y or z, list1, list2, list3)))

list1 = list(map(lambda x: x == 4 , df['other_simptomatology']))
list2 = list(map(lambda x: x == 4 , df['other_simptomatology_3']))
list3 = list(map(lambda x: x == 4 , df['other_simptomatology_4']))
df['mialgies'] = list(map(lambda x: 1 if x else 2, 
                               map(lambda x, y, z: x or y or z, list1, list2, list3)))
for field in other_simptomatology_fields:
    df = df.drop(field, axis=1)
list(df.columns)<jupyter_output><empty_output><jupyter_text>Positive Correlations Groups<jupyter_code>from collections import defaultdict
def positive_correlations_group(df):
    cor = df.corr()
    positive_correlations = defaultdict(lambda: [])
    for c in df.columns:
        series = cor[[c]].drop(c)
        series = series.dropna()
        series = series.transpose()
        var_corr_group = [ col for col in series.columns if series[col].item() >= 0.8]
        positive_correlations[c] = var_corr_group
    return positive_correlations<jupyter_output><empty_output><jupyter_text>Negative Correlation Groups<jupyter_code>from collections import defaultdict
def negative_correlations_group(df):
    cor = df.corr()
    negative_correlations = defaultdict(lambda: [])
    for c in df.columns:
        series = cor[[c]].drop(c)
        series = series.dropna()
        series = series.transpose()
        var_corr_group = [ col for col in series.columns if series[col].item() <= -0.8]
        negative_correlations[c] = var_corr_group
    return negative_correlations<jupyter_output><empty_output><jupyter_text>Now we will make the groups of the variables that have a strong correlation in the same direction.So first we will do grups for positive correations and then for negative ones.<jupyter_code>def purgue(el, correlations):
    dd = defaultdict(lambda x : [])
    for e in el:
        dd[e] = correlations[e]
    return dd<jupyter_output><empty_output><jupyter_text>All this drops are in orther to simplify this variables that are represented in groups<jupyter_code>df = df.drop('tachypnea_first', axis = 1)
df = df.drop('auscult_first', axis = 1)
df = df.drop('smell_first', axis = 1)
df = df.drop('ocular_first', axis = 1)
df = df.drop('adenopathies', axis = 1)
df = df.drop('splenomegaly', axis = 1)
df = df.drop('taste_smell', axis = 1)
df = df.drop('confusion', axis = 1)
df = df.drop('hypotonia', axis = 1)
df = df.drop('cxr2', axis = 1)
df = df.drop('rash', axis = 1)
df = df.drop('pcr_result', axis = 1)
df = df.drop('antigenic_result', axis = 1)
df = df.drop('smell', axis = 1)
df = df.drop('odynophagia_first', axis = 1)
df = df.drop('dyspnea_first', axis = 1)
df = df.drop('vomiting', axis = 1)
df = df.drop('sero_response', axis = 1)
df = df.drop('skin_first', axis = 1)
df = df.drop('seizures', axis = 1)
df = df.drop('cough_first', axis = 1)
df = df.drop('fatigue_first', axis = 1)
df = df.drop('conj_first', axis = 1)
positive_correlations = positive_correlations_group(df)
el1 = list(filter(lambda x: len(positive_correlations[x]) != 0, positive_correlations))
purgue(el1, positive_correlations)
df = df.drop('inflam_oral', axis = 1)
df = df.drop('irritability_first', axis = 1)
negative_correlations = negative_correlations_group(df)
el2 = list(filter(lambda x: len(negative_correlations[x]) != 0, negative_correlations))
purgue(el2, negative_correlations)
len(df.columns)
df<jupyter_output><empty_output><jupyter_text>Now that we have less variables, we do a heatmap again<jupyter_code>import seaborn as sns
sns.heatmap(df.corr(), vmin=-1, vmax=1, center= 0)<jupyter_output><empty_output><jupyter_text>Still seeing nothing<jupyter_code>check_values = {}
for col in df.columns:
    check_values[col] = get_population(df,df[col])
check_values
def last_diagnosis_code(df):
    diagnosis = df[['final_diagnosis_code']]
    new_df = df.drop('final_diagnosis_code', axis=1)
    new_df['final_diagnosis_code'] = diagnosis
    return new_df
df = last_diagnosis_code(df)
df<jupyter_output><empty_output><jupyter_text>We're gonna drop all the variables that have a ot of NaN values because we thing that don't provide so much information<jupyter_code>for col in filter(lambda x : check_values[x][np.nan] > 1600, df.columns):
    df = df.drop(col, axis = 1)
len(df.columns)
for col in filter(lambda x : check_values[x][np.nan] > 1500, df.columns):
    df = df.drop(col, axis = 1)
len(df.columns)<jupyter_output><empty_output><jupyter_text>We make an algorithm to constract Bayesian Nets with random variablesand get the best model of all the iterations.<jupyter_code>import random
from pomegranate import *
import numpy
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import networkx as nx

def get_rnd_cols(df, num_vars = 10):
    df_cols = list(df.columns)
    columns = []
    i = 0
    while i < num_vars:
        num = random.randint(0,len(df_cols)-1)
        if df_cols[num] not in columns:
            columns.append(df_cols[num])
            i += 1
    return df[columns]
    

def random_bayes(training_x, training_y):
    training = training_x.join(training_y)
    model = BayesianNetwork.from_samples(training, algorithm='exact')
    return model

def construct_test_set(X_test):
    to_predict = []
    for test in X_test.values.tolist():
        test.append(None)
        to_predict.append(test)
    return to_predict

def get_prediction_result(results, expected, value):
    score = 0;
    for i,result in enumerate(results):
        if result[-1] == value and result[-1] == expected[i]:
            score +=1
    number_of_values = len([x for x in expected if x == value])
    return score / number_of_values

def get_general_prediction_result(results, expected):
    score = 0;
    for i,result in enumerate(results):
        if result[-1] == expected[i]:
            score +=1
    number_of_values = len(expected)
    return score / number_of_values

def delete_half_negative(df, negative_cases):
    half_negative = negative_cases / 2
    for i in range(half_negative):
        df.sample()
    
def draw_bayesian_network(graph):
    G = nx.DiGraph()
    for i,node in enumerate(graph):
        G.add_node(i)
    for i,node in enumerate(graph):
        for edge in node:
            G.add_edge(edge,i)
    nx.draw(G, with_labels=True, arrowsize=10)
    plt.show()

best_result = 0
best_pos = 0
best_ns = 0
best_neg=0
df = df.dropna(subset=['final_diagnosis_code'])
y = df['final_diagnosis_code']
df = df.drop('final_diagnosis_code', axis = 1)
for i in range(1000):
    X = get_rnd_cols(df)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
    X_test.columns
    model = random_bayes(X_train, y_train)
    to_predict = construct_test_set(X_test)
    to_predict
    try:
        results = model.predict(to_predict)
    except ValueError:
        continue
    if get_general_prediction_result(results, y_test.values.tolist()) >= best_result:
        best_columns = X.columns
        best_result = get_general_prediction_result(results, y_test.values.tolist())
        best_pos = get_prediction_result(results, y_test.values.tolist(),1)
        best_ns = get_prediction_result(results, y_test.values.tolist(),2)
        best_neg = get_prediction_result(results, y_test.values.tolist(),3)
        best_model = model
print(best_columns)
draw_bayesian_network(best_model.structure)
print("General prediction result")
print(best_result)
print("Positive prediction result")
print(best_pos)
print("Don't know prediction result")
print(best_ns)
print("Negative prediction result")
print(best_neg)<jupyter_output>Index(['sympt_epi', 'odynophagia', 'artralgies', 'sex', 'ausc_resp',
       'home_confirmed', 'headache_first', 'fever', 'resp',
       'nasal_congestion'],
      dtype='object')<jupyter_text>This up here is the best model of a Bayesian Network that we got.It predicts the 87% of the cases. We can see that predicts the 97% of negative cases and the 36% of the positive.As a conclusion, I thing that the quality of the data it's not so good. Starting from the amount of NaN values that it contains, from the amount of columns in the dataset and the one that I think is the most important, ther are so much negateve cases and to feew other cases.I think that with a good selected set of variables, a bayesian net is a good way to go. Due to the charasteristics of the problem, I thing is a very good aproach. From a couple of simpthomps we infeer the result of the goal variable.But also it has limitations, and that is mainly because the algorithm that constructs the bayesian net is of exponential cost, making the number of variables that you can work with limitated. In fact, in the docs of the library that we used, advises us that we should not try to construct a net of more than 25 variables.<jupyter_code>#json = model.to_json()
#f = open("./bayes_nets/model2.txt", "w")
#f.write(json)
#json
sample = df.sample().join(y)
sample
column_samples = sample[best_columns]
list_sample = column_samples.values.tolist()
list_sample[0].append(None)
best_model.predict(list_sample)<jupyter_output><empty_output><jupyter_text>We try to open the model that we serialized into a file and do some predicts but it didn't work :(<jupyter_code>f = open("./bayes_nets/model2.txt", "r")
json = f.read()
same_model = BayesianNetwork.from_json(json)
draw_bayesian_network(same_model.structure)
same_model.predict(list_sample)
positive_cases = len([x for x in df.values.tolist() if x[-1] == 1])
dk_cases = len([x for x in df.values.tolist() if x[-1] == 2])
negative_cases = len([x for x in df.values.tolist() if x[-1] == 3])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13368171598711795, 'fraction_numerical': 0.017439387494683115, 'mean_word_length': 4.926539431040691, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '407018', 'license': 'MIT', 'n_tokens_mistral': 6153, 'n_tokens_neox': 5872, 'n_words': 1471, 'path': 'bayes.ipynb', 'repo_name': 'Oriolac/BitsXLaMarato', 'size': '16457'}