<jupyter_start><jupyter_text>Predicting patient perception / how they feel towards themselves(their anxiety/pain ..) from: the other EQ-5D-5L scores and sociodemographic features<jupyter_code>import savReaderWriter as spss
import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt 
import seaborn as sns
import matplotlib.style as style
style.use('seaborn')

import ipywidgets as widgets
from ipywidgets import interact
from ipywidgets import interactive, Layout, HTML

from imblearn.over_sampling import SMOTE
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.tree import DecisionTreeClassifier,_tree
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn import model_selection
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
#from sklearn.metrics import classification_report 
#from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler


from imblearn.over_sampling import SMOTE
import itertools

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore")
import savReaderWriter as spss
import pandas as pd


raw_data = spss.SavReader('p29_data.sav', returnHeader = True)
raw_data_list = list(raw_data)
df = pd.DataFrame(raw_data_list)
with spss.SavReader('p29_data.sav', ioUtf8 = True) as reader:
    df = pd.DataFrame(reader.all(), columns = [s.decode('CP1252') for s in reader.header])
        
    
#prom= df[[ 'WEIGHT', 'GENDER', 'AGER','AGE','E1', 'E2', 'E3', 'E4', 'E5', 'E6']].copy()
eq5d= df[[ 'WEIGHT', 'GENDER', 'AGE','E1', 'E2', 'E3', 'E4', 'E5', 'E6']].copy()
eq5d.columns = ['WEIGHT', 'GENDER', 'AGE','MOBILITY', 'SELF CARE', 'USUAL ACTIVITY', 'PAIN', 'ANXIETY', 'HEALTH SCALE']


eq= eq5d[['MOBILITY', 'SELF CARE', 'USUAL ACTIVITY', 'PAIN', 'ANXIETY']].copy()
new = eq5d.copy()
new['GENERAL HEALTH'] = eq.sum(axis=1)
#eq5d.head(3)    
dff= df[[ 'WEIGHT', 'GENDER', 'AGE','OCC', 'HHCMP5', 'MARITAL','EDU3','INC4','E1', 'E2', 'E3', 'E4', 'E5', 'E6']].copy()




dff.columns = ['WEIGHT', 'GENDER', 'AGE',
               'OCCUPATION','FAMILY_NUMBER','MARITAL','EDUCATION','INCOME',
               'MOBILITY', 'SELF CARE', 'USUAL ACTIVITY', 'PAIN', 'ANXIETY', 'HEALTH SCALE',
               ]

dff['GENERAL HEALTH'] = eq.sum(axis=1)
dff
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):

    """
    This function prints and plots the confusion matrix.`.
    """

    #style.use('default')

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.tight_layout()
from collections import Counter
from matplotlib import pyplot<jupyter_output><empty_output><jupyter_text>labels is highly imbalanced<jupyter_code>Label= widgets.Dropdown(options=['MOBILITY', 'SELF CARE','USUAL ACTIVITY', 'PAIN', 'ANXIETY'], value='ANXIETY')

@interact(Label = Label)
def pltt(Label):

    y = new[Label]
    x = new


    counter = Counter(y)
    for k,v in counter.items():
        per = v / len(y) * 100
        print('Class=%d, n=%d (%.3f%%)' % (k, v, per))
    # plot the distribution
    pyplot.bar(counter.keys(), counter.values())
    pyplot.show()

Label= widgets.Dropdown(options=['MOBILITY', 'SELF CARE','USUAL ACTIVITY', 'PAIN', 'ANXIETY'], value='ANXIETY')

@interact(Label = Label)
def random_forrest(Label):
    x = new.drop(Label,axis = 1)
    y = new[Label]
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=42,shuffle = True)

    random_forest = RandomForestClassifier(n_estimators=100, n_jobs=-1)

    clf = random_forest.fit(x_train,y_train)

    y_pred = clf.predict(x_test)
    
    
    
    print ("Number of training observations ; ", len(y_train))
    print ("________________________")
    print ("Number of test observations ; ", len(y_test))
    print ("________________________")

    print ("cross Testing result========")
    scores = cross_val_score(clf, x_test, y_test, cv=5)
    print (scores)
    print ("________________________")
    print ("Testing result mean ========")
    print (scores.mean())
    print ("________________________")


    cnf_matrix = confusion_matrix(y_test,y_pred)
    print ("Printing confusion Matrix ========")
    plot_confusion_matrix(cnf_matrix,classes=[1,2,3,4,5])
    print ("________________________")<jupyter_output><empty_output><jupyter_text>Results after oversampling (SMOTE)<jupyter_code>oversample = SMOTE()

Label= widgets.Dropdown(options=['MOBILITY', 'SELF CARE','USUAL ACTIVITY', 'PAIN', 'ANXIETY'], value='ANXIETY')

@interact(Label = Label)
def pltt(Label):



    y = new[Label]
    x = new


    oversample = SMOTE()
    x_new, y_new = oversample.fit_resample(x, y)
    # summarize distribution
    counter = Counter(y_new)
    for k,v in counter.items():
        per = v / len(y_new) * 100
        print('Class=%d, n=%d (%.3f%%)' % (k, v, per))
    # plot the distribution
    pyplot.bar(counter.keys(), counter.values())
    pyplot.show()

Label= widgets.Dropdown(options=['MOBILITY', 'SELF CARE','USUAL ACTIVITY', 'PAIN', 'ANXIETY'], value='PAIN')

@interact(Label = Label)
def random_forrest(Label):
    x = new.drop(Label,axis = 1)
    y = new[Label]
    
    x_train, x_test, y_train, y_test = train_test_split(x,y, test_size=0.4, random_state=42,shuffle = True)
    x_new, y_new = oversample.fit_resample(x_train, y_train)

    random_forest = RandomForestClassifier(n_estimators=100, n_jobs=-1)

    clf = random_forest.fit(x_new, y_new)

    y_pred = clf.predict(x_test)
    
    
    print ("Number of training observations ; ", len(y_new))
    print ("________________________")
    print ("Number of test observations ; ", len(y_test))
    print ("________________________")
        
    print ("cross Testing result========")
    scores = cross_val_score(clf, x_test, y_test, cv=5)
    print (scores)
    print ("________________________")
    print ("Testing result mean ========")
    print (scores.mean())
    print ("________________________")


    cnf_matrix = confusion_matrix(y_test,y_pred)
    print ("Printing confusion Matrix ========")
    plot_confusion_matrix(cnf_matrix,classes=[1,2,3,4,5])
    print ("________________________")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14605282147496032, 'fraction_numerical': 0.012122961466301053, 'mean_word_length': 4.230188679245283, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '239452', 'license': 'MIT', 'n_tokens_mistral': 2477, 'n_tokens_neox': 2356, 'n_words': 533, 'path': 'app.ipynb', 'repo_name': 'Abanobbb/eq', 'size': '6929'}