<jupyter_start><jupyter_text>Data Visualization<jupyter_code>#Collect all columns having Missing values and at what percentage
columns_with_na=[features for features in df_copy.columns if df_copy[features].isnull().sum()>0]

for feature in columns_with_na:
    print(feature, np.round(df_copy[feature].isnull().mean(), 4),  ' % missing values')
for feature in columns_with_na:
    data = df_copy.copy()
    data[feature] = np.where(data[feature].isnull(), 1, 0)
    data.groupby(feature)['Outcome'].value_counts().plot.bar()
    plt.title(feature)
    plt.show()
#Get all the columns having numerical values 
num_data = [features for features in df_copy.columns if df_copy[features].dtypes != 'O']
print(f'Number of numerical columns is {len(num_data)}')

df_copy[num_data].head()
# Check if any numerical columns are discrete
discrete_columns = [feature for feature in df_copy.columns if len(df_copy[feature].unique()) < 20 and feature not in ['Outcome'] ]
print(f'Number of discrete Columns is {len(discrete_columns)}')
for i in discrete_columns:
    print(f'{i} has {len(df_copy[i].unique())} discrete values')

df_copy[discrete_columns].head()
#Check distribution of the discrete data
for feature in discrete_columns:
    dt=df_copy.copy()
    dt.groupby(feature)['Outcome'].value_counts().plot.bar()
    plt.xlabel(feature)
    plt.ylabel('Outcome')
    plt.title(feature)
    plt.show()
# Check if any numerical columns are continous
continous_columns = [feature for feature in num_data if feature not in discrete_columns and feature not in ['Outcome'] ]
print(f'Number of Continous Columns is {len(continous_columns)}')

df_copy[continous_columns].head()
#Check distribution of the Continous data
for feature in continous_columns:
    dt = df_copy.copy()
    dt[feature].hist(bins=25)
    plt.xlabel(feature)
    plt.ylabel("Count")
    plt.title(feature)
    plt.show()<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>#Collect all columns having Missing values and at what percentage
columns_with_na=[features for features in df_copy.columns if df_copy[features].isnull().sum()>0]

for feature in columns_with_na:
    print(feature, np.round(df_copy[feature].isnull().mean(), 4),  ' % missing values')
#Replace Nan Values with median of that column and make new column as df_copy[feature_nan]. Give it value 1 is Nan is present else 0 
for feature in columns_with_na:
    median_value = df_copy[feature].mean()
    df_copy[feature+'nan'] = np.where(df_copy[feature].isnull(), 1, 0)
    df_copy[feature].fillna(median_value, inplace = True)
    
df_copy[columns_with_na].isnull().sum()
df_copy.head()
# Remove any duplicate entry
final_df =df_copy.loc[:,~df_copy.columns.duplicated()]
final_df.shape
# Model Building
from sklearn.model_selection import train_test_split
X = df.drop(columns='Outcome')
y = df['Outcome']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=0)<jupyter_output><empty_output><jupyter_text>Naive Bayes<jupyter_code>from sklearn.naive_bayes import GaussianNB
classifier = GaussianNB()

classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
y_pred
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, y_pred)
print(cm)
print(accuracy_score(y_test, y_pred))<jupyter_output>[[93 14]
 [18 29]]
0.7922077922077922<jupyter_text>Random Forest<jupyter_code># Creating Random Forest Model
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV

clf = RandomForestClassifier()

n_estimators = [10, 20, 30, 50, 100]
max_depth = [2, 3, 5, 7, 10]

# Define the grid of hyperparameters to search
hyperparameter_grid = {
    'n_estimators': n_estimators,
    'max_depth':max_depth,
    }
random_cv = RandomizedSearchCV(estimator=clf,param_distributions=hyperparameter_grid,
                               cv=5, n_iter=5,scoring = 'neg_mean_absolute_error',n_jobs = 4,verbose = 5,
                               return_train_score = True,random_state=42)
random_cv.fit(X_train,y_train)
random_cv.best_estimator_
clf = RandomForestClassifier(max_depth=10, n_estimators=50)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
y_pred
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, y_pred)
print(cm)
print(accuracy_score(y_test, y_pred))<jupyter_output>[[96 11]
 [17 30]]
0.8181818181818182<jupyter_text>Decision Tree<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import GridSearchCV
classifier = DecisionTreeClassifier()

from scipy.stats import randint
from sklearn.model_selection import RandomizedSearchCV

parameters = {"max_depth":[2,3, None],
              "max_features":randint(1,9),
              "min_samples_leaf":randint(1,8),
              "criterion": ["gini", "entropy"]
}

tree_cv = RandomizedSearchCV(classifier, parameters, cv = 5)
tree_cv.fit(X_train, y_train)
tree_cv.best_estimator_
classifier = DecisionTreeClassifier(max_depth=2, max_features=6, min_samples_leaf=7)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
y_pred
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, y_pred)
print(cm)
print(accuracy_score(y_test, y_pred))
# Creating a pickle file for the classifier
import pickle
filename = 'diabetes-model.pkl'
pickle.dump(clf, open(filename, 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11062109518559353, 'fraction_numerical': 0.019294377067254686, 'mean_word_length': 5.924936386768448, 'pattern_counts': {'":': 4, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '54153', 'license': 'MIT', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1835, 'n_words': 419, 'path': 'Notebooks/DiabetesNotebook.ipynb', 'repo_name': 'srisatya1197/HealthCheck', 'size': '5442'}