<jupyter_start><jupyter_text>Data Preprocessing and Architecture set-up<jupyter_code>import os
path = "/content/drive/MyDrive/Intuition2021"
vaers_data_dir = os.path.join(path, '2021VAERSDATA.csv')
vaers_symptom_dir = os.path.join(path, '2021VAERSSYMPTOMS.csv')
vaers_vax_dir = os.path.join(path, '2021VAERSVAX.csv')
import chardet
rawdata = open(vaers_data_dir, 'rb').read()
result = chardet.detect(rawdata)
charenc = result['encoding']
print(charenc)
import pandas as pd 
vaers_data = pd.read_csv(vaers_data_dir, encoding='Windows-1252')
vaers_symptom = pd.read_csv(vaers_symptom_dir)
vaers_vax = pd.read_csv(vaers_vax_dir)
vaers_data.head()
vaers_symptom.head(10)
vaers_vax.head()
vaers_symptom.dtypes
vaers_symptom[vaers_symptom['VAERS_ID'] == 916741]
print(vaers_symptom['VAERS_ID'].diff(periods=-1))
tmp = vaers_symptom['VAERS_ID'].diff(periods=-1)
tmp2 = []
for i in range(len(tmp)):
  if tmp[i] == 0:
    tmp2.append(i)
vaers_symptom = vaers_symptom.drop(vaers_symptom.index[tmp2])
vaers_symptom
len(vaers_symptom) == len(vaers_data)
vaers_merge = pd.merge(vaers_data, vaers_symptom, on='VAERS_ID')
AGE_YRS, SEX,
vaers_merge.head(50)
len(vaers_merge)
vaers_vax.head(30)
len(vaers_vax[vaers_vax['VAX_TYPE'] == 'COVID19'])
vaers_vax = vaers_vax[vaers_vax['VAX_TYPE'] == 'COVID19']
vaers_vax
vaers_merge_f = pd.merge(vaers_vax, vaers_merge, on='VAERS_ID')
len(vaers_merge_f)
VAX_MANU, AGE_YRS, SEX, OTHER_MEDS, CUR_ILL, HISTORY, BIRTH_DEFECT, ALLERGIES, SYMPTOM1, SYMPTOM2, SYMPTOM3, SYMPTOM4, SYMPTOM5

vaers_merge.head(20)
 vaers_merge.isna().sum()
vaers_merge_f = vaers_merge_f[['VAX_MANU', 'AGE_YRS', 'SEX', 'OTHER_MEDS', 'CUR_ILL', 'HISTORY', 'BIRTH_DEFECT', 'ALLERGIES', 'SYMPTOM1', 'SYMPTOM2', 'SYMPTOM3', 'SYMPTOM4', 'SYMPTOM5']]
vaers_merge_f
len(vaers_merge_f.ALLERGIES.unique())
vaers_merge_f.BIRTH_DEFECT.unique()
len(vaers_merge_f.OTHER_MEDS.unique())
def age_to_numeric(x):
    if x=='No': return str(45)
    else: return x

def sex_to_numeric(x):
    if x=='F': return 1
    if x=='M':   return 0

def vax_manu_to_numeric(x):
  if x == 'UNKNOWN MANUFACTURER': return 0
  if x == 'MODERNA': return 1
  if x == 'PFIZER\\BIONTECH': return 2

def other_meds_to_numeric(x):
  if x == 'NaN': return 0
  if x == 'No': return 0
  if x == 'NKDA': return 0
  if x == 'None': return 0
  else: return 1

def curr_ill_to_numeric(x):
  if x == 'NaN': return 0
  if x == 'No': return 0
  if x == 'NKDA': return 0
  if x == 'None': return 0
  else: return 1

def history_to_numeric(x):
  if x == 'NaN': return 0
  if x == 'No': return 0
  if x == 'NKDA': return 0
  if x == 'None': return 0
  else: return 1

def birth_defect_to_numeric(x):
  if x == 'NaN': return 0
  if x == 'No': return 0
  if x == 'NKDA': return 0
  if x == 'None': return 0
  else: return 1

def allergies_to_numeric(x):
  if x == 'NaN': return 0
  if x == 'No': return 0
  if x == 'NKDA': return 0
  if x == 'None': return 0
  else: return 1

def symptom_to_numeric(x):
  if x == 'NaN': return 0
  if x == 'No': return 0
  if x == 'NKDA': return 0
  if x == 'None': return 0
  else: return 1




  

vaers_merge_f["SYMPTOM5"]
vaers_merge_f.AGE_YRS.isna().sum()
vaers_merge_f["VAX_MANU"] = vaers_merge_f["VAX_MANU"].fillna("No")
vaers_merge_f["SEX"] = vaers_merge_f["SEX"].fillna("No")
vaers_merge_f["OTHER_MEDS"] = vaers_merge_f["OTHER_MEDS"].fillna("No")
vaers_merge_f["CUR_ILL"] = vaers_merge_f["CUR_ILL"].fillna("No")
vaers_merge_f["HISTORY"] = vaers_merge_f["HISTORY"].fillna("No")
vaers_merge_f["BIRTH_DEFECT"] = vaers_merge_f["BIRTH_DEFECT"].fillna("No")
vaers_merge_f["ALLERGIES"] = vaers_merge_f["ALLERGIES"].fillna("No")
vaers_merge_f["SYMPTOM1"] = vaers_merge_f["SYMPTOM1"].fillna("No")
vaers_merge_f["SYMPTOM2"] = vaers_merge_f["SYMPTOM2"].fillna("No")
vaers_merge_f["SYMPTOM3"] = vaers_merge_f["SYMPTOM3"].fillna("No")
vaers_merge_f["SYMPTOM4"] = vaers_merge_f["SYMPTOM4"].fillna("No")
vaers_merge_f["SYMPTOM5"] = vaers_merge_f["SYMPTOM5"].fillna("No")
vaers_merge_f.head()
vaers_merge_f['vax_manu_num'] = vaers_merge_f['VAX_MANU'].apply(vax_manu_to_numeric)
vaers_merge_f['sex_num'] = vaers_merge_f['SEX'].apply(sex_to_numeric)
vaers_merge_f['other_meds_num'] = vaers_merge_f['OTHER_MEDS'].apply(other_meds_to_numeric)
vaers_merge_f['curr_ill_num'] = vaers_merge_f['CUR_ILL'].apply(curr_ill_to_numeric)
vaers_merge_f['history_num'] = vaers_merge_f['HISTORY'].apply(history_to_numeric)
vaers_merge_f['birth_defect_num'] = vaers_merge_f['BIRTH_DEFECT'].apply(birth_defect_to_numeric)
vaers_merge_f['allergies_num'] = vaers_merge_f['ALLERGIES'].apply(allergies_to_numeric)
vaers_merge_f['symptom1_num'] = vaers_merge_f['SYMPTOM1'].apply(symptom_to_numeric)
vaers_merge_f['symptom2_num'] = vaers_merge_f['SYMPTOM2'].apply(symptom_to_numeric)
vaers_merge_f['symptom3_num'] = vaers_merge_f['SYMPTOM3'].apply(symptom_to_numeric)
vaers_merge_f['symptom4_num'] = vaers_merge_f['SYMPTOM4'].apply(symptom_to_numeric)
vaers_merge_f['symptom5_num'] = vaers_merge_f['SYMPTOM5'].apply(symptom_to_numeric)

vaers_merge_f
vaers_merge_f["AGE_YRS"]
vaers_merge_f["AGE_YRS"] = vaers_merge_f["AGE_YRS"].fillna("No")
sum = 0
for row in vaers_merge_f.iterrows():
  if row[1][1] != "No":
    sum = sum + int(row[1][1])
  
vaers_merge_f['age_yrs'] = vaers_merge_f['AGE_YRS'].apply(age_to_numeric)
vaers_merge_f["age_yrs"] = pd.to_numeric(vaers_merge_f["age_yrs"])
vaers_merge_final = vaers_merge_f.drop(['VAX_MANU', 'AGE_YRS', 'SEX', 'OTHER_MEDS', 'CUR_ILL', 'HISTORY', 'BIRTH_DEFECT', 'ALLERGIES', 'SYMPTOM1', 'SYMPTOM2', 'SYMPTOM3', 'SYMPTOM4', 'SYMPTOM5'], axis=1)
vaers_merge_final.dtypes
first_col = vaers_merge_final.pop('age_yrs')
vaers_merge_final.insert(0, 'age_yrs', first_col)
vaers_merge_final
sample_dataset = vaers_merge_final.sample(n=300)
X = sample_dataset.iloc[:,: -5]
X = vaers_merge_final.iloc[:,: -5]
X = X.sample(n = 300)
Y = sample_dataset.iloc[:,-5: ]
Y = vaers_merge_final.iloc[:,-5: ]
Xz = X.to_numpy()
Yz = Y.to_numpy()
n_inputs, n_outputs = Xz.shape[1], Yz.shape[1]
model = get_model(n_inputs, n_outputs)
model.fit(Xz, Yz, verbose=0, epochs=100)
vaers_merge_final = vaers_merge_final.dropna()
vaers_merge_final
from numpy import mean
from numpy import std
from sklearn.datasets import make_multilabel_classification
from sklearn.model_selection import RepeatedKFold
from keras.models import Sequential
from keras.layers import Dense
from sklearn.metrics import accuracy_score
import numpy as np
import tensorflow as tf
def get_model(n_inputs, n_outputs):
  model = Sequential()
  model.add(Dense(20, input_dim=n_inputs, kernel_initializer='he_uniform', activation='relu'))
  model.add(Dense(n_outputs, activation='sigmoid'))
  model.compile(loss='categorical_crossentropy', optimizer='adam')
  return model
def evaluate_model(model, X, y):
  results = list()
  results_loss = list()
  n_inputs, n_outputs = X.shape[1], y.shape[1]
# define evaluation procedure
  cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
# enumerate folds
  for train_ix, test_ix in cv.split(X):
   # prepare data
    X_train, X_test = X[train_ix], X[test_ix]
    y_train, y_test = y[train_ix], y[test_ix]
    # X_train, y_train= X, y
    model_f = model
    model_f.fit(X_train, y_train, verbose=0, epochs=1)
    yhat = model_f.predict(X_train)
    yhat = yhat.round()
    acc = accuracy_score(y_train, yhat)
    count_tot = 0
    count_cor = 0
    for i in range(len(yhat)):
      for j in range(len(yhat[i])):
        if(yhat[i][j]==y_train[i][j]):
          count_cor+=1.0
        count_tot += 1.0
    cce = tf.keras.losses.CategoricalCrossentropy()
    loss = cce(y_train, yhat).numpy()
    acc = count_cor/count_tot
    results.append(acc)
    results_loss.append(loss)
  return str(count_cor/count_tot)

def evaluate_only(model, X, y):
  yhat = model.predict(X)
  yhat = yhat.round()
  acc = accuracy_score(y, yhat)
  print('>%.3f' % acc)
  results.append(acc)
  return results

def getWeights(model):
    return model.get_weights()

def setWeights(model,weight):
    model.set_weights(weight)
class Aggregator():
    
    def __init__(self):
        self.wB1=0.33
        self.wB2=0.33
        self.wB3=0.33
    
    def aggregate(self,delta, ):
        delta=np.array(delta)
        tmp = s/12
        temp=(self.wB1*np.array(B1) + self.wB2*np.array(B2) + self.wB3*np.array(B3))
        delta = tmp
        
        return delta
sample_dataset = vaers_merge_final.sample(n=300)
X = sample_dataset.iloc[:,: -5]
Y = sample_dataset.iloc[:,-5: ]
Xz = X.to_numpy()
Yz = Y.to_numpy()
X = vaers_merge_final.iloc[:,: -5]
Y = vaers_merge_final.iloc[:,-5: ]
Xz = X.to_numpy()
Yz = Y.to_numpy()
s1 = vaers_merge_final[0:300][:]
s2 = vaers_merge_final[300:600][:]
s3 = vaers_merge_final[600:900][:]
s4 = vaers_merge_final[900:1200][:]
s5 = vaers_merge_final[1200:1500][:]
s6 = vaers_merge_final[1800:2100][:]
s7 = vaers_merge_final[2100:2400][:]
s8 = vaers_merge_final[2400:2700][:]
s9 = vaers_merge_final[2700:3000][:]
s10 = vaers_merge_final[3000:3300][:]
s11 = vaers_merge_final[3300:3600][:]
s12 = vaers_merge_final[3600:3900][:]
sG = vaers_merge_final[3900: ][:]

X1 = s1.iloc[:,: -5]
Y1= s1.iloc[:,-5: ]
X2 = s2.iloc[:,: -5]
Y2 = s2.iloc[:,-5: ]
X3 = s3.iloc[:,: -5]
Y3 = s3.iloc[:,-5: ]
X4 = s4.iloc[:,: -5]
Y4= s4.iloc[:,-5: ]
X5 = s5.iloc[:,: -5]
Y5 = s5.iloc[:,-5: ]
X6 = s6.iloc[:,: -5]
Y6 = s6.iloc[:,-5: ]
X7 = s7.iloc[:,: -5]
Y7 = s7.iloc[:,-5: ]
X8 = s8.iloc[:,: -5]
Y8= s8.iloc[:,-5: ]
X9 = s9.iloc[:,: -5]
Y9 = s9.iloc[:,-5: ]
X10 = s10.iloc[:,: -5]
Y10 = s10.iloc[:,-5: ]
X11 = s11.iloc[:,: -5]
Y11= s11.iloc[:,-5: ]
X12 = s12.iloc[:,: -5]
Y12= s12.iloc[:,-5: ]
XG = sG.iloc[:,: -5]
YG= sG.iloc[:,-5: ]



test_features = X1.to_numpy()
test_labels = Y1.to_numpy()
X2z = X2.to_numpy()
Y2z = Y2.to_numpy()
X3z = X3.to_numpy()
Y3z = Y3.to_numpy()
X4z = X4.to_numpy()
Y4z = Y4.to_numpy()
X5z = X5.to_numpy()
Y5z = Y5.to_numpy()
X6z = X6.to_numpy()
Y6z = Y6.to_numpy()
X7z = X7.to_numpy()
Y7z = Y7.to_numpy()
X8z = X8.to_numpy()
Y8z = Y8.to_numpy()
X9z = X9.to_numpy()
Y9z = Y9.to_numpy()
X10z = X10.to_numpy()
Y10z = Y10.to_numpy()
X11z = X11.to_numpy()
Y11z = Y11.to_numpy()
X12z = X12.to_numpy()
Y12z = Y12.to_numpy()
XGz = XG.to_numpy()
YGz = YG.to_numpy()


n1_inputs, n1_outputs = X1z.shape[1], Y1z.shape[1]
n2_inputs, n2_outputs = X2z.shape[1], Y2z.shape[1]
n3_inputs, n3_outputs = X3z.shape[1], Y3z.shape[1]
n4_inputs, n4_outputs = X4z.shape[1], Y4z.shape[1]
n5_inputs, n5_outputs = X5z.shape[1], Y5z.shape[1]
n6_inputs, n6_outputs = X6z.shape[1], Y6z.shape[1]
n7_inputs, n7_outputs = X7z.shape[1], Y7z.shape[1]
n8_inputs, n8_outputs = X8z.shape[1], Y8z.shape[1]
n9_inputs, n9_outputs = X9z.shape[1], Y9z.shape[1]
n10_inputs, n10_outputs = X10z.shape[1], Y10z.shape[1]
n11_inputs, n11_outputs = X11z.shape[1], Y11z.shape[1]
n12_inputs, n12_outputs = X12z.shape[1], Y12z.shape[1]
nG_inputs, nG_outputs = XGz.shape[1], YGz.shape[1]


user1_model = get_model(n1_inputs, n1_outputs)
model2 = get_model(n2_inputs, n2_outputs)
model3 = get_model(n3_inputs, n3_outputs)
model4 = get_model(n4_inputs, n4_outputs)
model5 = get_model(n5_inputs, n5_outputs)
model6 = get_model(n6_inputs, n6_outputs)
model7 = get_model(n7_inputs, n7_outputs)
model8 = get_model(n8_inputs, n8_outputs)
model9 = get_model(n9_inputs, n9_outputs)
model10 = get_model(n10_inputs, n10_outputs)
model11 = get_model(n11_inputs, n11_outputs)
model12 = get_model(n12_inputs, n12_outputs)
federated_model = get_model(nG_inputs, nG_outputs)
model_all = get_model(n12_inputs, n12_outputs)<jupyter_output><empty_output><jupyter_text>**Evaluating the model on a singel user data**<jupyter_code># evaluating model trained with data obtained from a single user  on test data.
result_user_1 = evaluate_model(user1_model, test_features, test_labels) 
print("Checking accuracy on Test Data")
print(result_user_1)<jupyter_output>Checking accuracy on Test Data
0.5228571428571429<jupyter_text>**-----------------------------------------------------**<jupyter_code>test_features.shape<jupyter_output><empty_output><jupyter_text>**Evaluating horizontal federated trained model**<jupyter_code># evaluating federated model on test data.
result_federated_model = evaluate_model(federated_model, test_features, test_labels) 
print("Checking accuracy on Test Data")
print(result_federated_model)<jupyter_output>Checking accuracy on Test Data
0.61<jupyter_text>**-----------------------------------------------------** **Evaluting model trained on entire dataset**<jupyter_code># evaluating train model on complete test data.
result_full_model = evaluate_model(model_all, test_features, test_labels) 
print("Checking accuracy on Test Data")
print(result_full_model)<jupyter_output>Checking accuracy on Test Data
0.6365217391304347<jupyter_text>**-----------------------------------------------------**<jupyter_code># Training 12 clients
res1 = evaluate_model(model1, X1z, Y1z) #evaluation and training
res2 = evaluate_model(model2, X2z, Y2z)
res3 = evaluate_model(model3, X3z, Y3z)
res4 = evaluate_model(model4, X4z, Y4z) #evaluation and training
res5 = evaluate_model(model5, X5z, Y5z)
res6 = evaluate_model(model6, X6z, Y6z)
res7 = evaluate_model(model7, X7z, Y7z) #evaluation and training
res8 = evaluate_model(model8, X8z, Y8z)
res9 = evaluate_model(model9, X9z, Y9z)
res10 = evaluate_model(model10, X10z, Y10z) #evaluation and training
res11 = evaluate_model(model11, X11z, Y11z)
res12 = evaluate_model(model12, X12z, Y12z)
res12 = evaluate_model(modelG, XGz, YGz)

print("results: ")
print(res1[-1])
print(res2[-1])
print(res3[-1])
print(res4[-1])
print(res5[-1])
print(res6[-1])
print(res7[-1])
print(res8[-1])
print(res9[-1])
print(res10[-1])
print(res11[-1])
print(res12[-1])
print(resG[-1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16761779338601113, 'fraction_numerical': 0.051287679250804796, 'mean_word_length': 5.552732502396932, 'pattern_counts': {'":': 1, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '353298', 'license': 'MIT', 'n_tokens_mistral': 6253, 'n_tokens_neox': 6004, 'n_words': 1026, 'path': 'notebooks/Intuition2020.ipynb', 'repo_name': 'guptajay/iNTUition-v7.0', 'size': '13668'}