<jupyter_start><jupyter_text>Data Transformation<jupyter_code>import pandas as pd
import json
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn import tree
from sklearn.svm import LinearSVC
from pandas.core.frame import DataFrame
import numpy as np
from sklearn.linear_model import LogisticRegression
def disease_dict(column):
    count=0
    disease_set = set()
    disease_dict = {}
    for line in column:
        if line==None:
            continue
        else:
            disease_set.add(line)
    for item in disease_set:
        count += 1
        disease_dict[item]=count
    return disease_dict

# Import dataset and count the null value
data = pd.read_csv('final_result.csv')
data = data.drop(columns=['DAYSLAG_DIS_PREVDIS','SEQ_NO','SEQ_NO_1'])
data['PBSDISEASEGROUP'].isnull().value_counts()
##If does not already got the disease dictonary run this code
# disease_list = disease_dict(data['PBSDISEASEGROUP'])
# with open('final_disease_dic.json','w') as f:
#     json.dump(disease_list, f)
#Otherwise, just open the dictonary
with open('final_disease_dic.json','r', encoding='utf-8') as f:
    disease_list = json.load(f)
#clean dataset
#replace disease with dictonary
data.replace(disease_list,inplace = True)

# sort
data_sorted = data.sort_values(
    ['MASTERPATIENTID', 'DISPENSECALENDARDATE'], ascending=[1, 1])

# remove duplicate
data_sorted.drop_duplicates(
    subset=['MASTERPATIENTID',
        'PBSDISEASEGROUP'],
    keep='first',
    inplace=True)

new_data = data_sorted.drop(columns=['DISPENSECALENDARDATE'])
#group by patient id and make list of disease
data_new = new_data.groupby(['MASTERPATIENTID'])['PBSDISEASEGROUP'].apply(list)
#number of distinct patient
data_new.shape
#transform data set
transform = data_sorted.drop(columns=['DISPENSECALENDARDATE'])
transform['count1'] = 1
transform.set_index('MASTERPATIENTID',inplace = True)
transform = transform.set_index('PBSDISEASEGROUP',append = True)
transform = pd.Series(transform.values.reshape(len(transform)),index = transform.index)
transform = transform.unstack()
transform = transform.fillna(0)

##deleting the last value y
i=0
for id in data_new.keys():
    transform[data_new[id][-1]][i] = 0.00000
    i+=1

without_zero = transform.ix[~(transform==0).all(axis=1),:]
# y dataframe
y_list = []
for id in data_new.keys():
    if len(data_new[id]) == 1:
        continue
    y_list.append(data_new[id][-1])
c={"y" : y_list}
y_result = pd.DataFrame(c)

# adding y value to the dataset
new_transform = without_zero
new_transform = new_transform.reset_index(drop=True)
new_transform = pd.concat([new_transform,y_result], axis=1)
x = new_transform.iloc[:, :-1]
y = new_transform.iloc[:, -1]
#split into train data and test data 20%
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>clf_lr = LogisticRegression(solver='lbfgs',multi_class='multinomial')
clf_lr = clf_lr.fit(x_train, y_train)
y_predict_clf_lr_prob = clf_lr.predict_proba(x_test)
y_predict_clf_lr = clf_lr.predict(x_test)
accuracy_score(y_test, y_predict_clf_lr)<jupyter_output><empty_output><jupyter_text>Linear SVC<jupyter_code>clf_linearSVC = LinearSVC(multi_class='ovr')
clf_linearSVC = clf_linearSVC.fit(x_train, y_train)
y_predict_linearSVC = clf_linearSVC.predict(x_test)
accuracy_score(y_test, y_predict_linearSVC)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11678004535147392, 'fraction_numerical': 0.008786848072562359, 'mean_word_length': 6.086345381526105, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '654002', 'license': 'BSD-Source-Code', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1251, 'n_words': 251, 'path': 'lr_linearSVC.ipynb', 'repo_name': 'muhammadatifsajjad/MS-Data-Science-Project-Predicting-Patient-Outcomes', 'size': '3528'}