<jupyter_start><jupyter_text>Preprocessing<jupyter_code>print('Number of null values in each column')
print(raw_data.isnull().sum())

print()
columns_with_null_values = []
for i in range(len(raw_data.columns)):
    if raw_data[raw_data.columns[i]].isnull().sum() > 0:
        columns_with_null_values.append(raw_data.columns[i])
print('Columns with null values: ', columns_with_null_values)
raw_data.info()
for i in range(len(columns_with_null_values)):
    print(columns_with_null_values[i], raw_data[columns_with_null_values[i]].unique())<jupyter_output>education [ 4.  2.  1.  3. nan]
cigsPerDay [ 0. 20. 30. 23. 15.  9. 10.  5. 35. 43.  1. 40.  3.  2. nan 12.  4. 18.
 25. 60. 14. 45.  8. 50. 13. 11.  7.  6. 38. 29. 17. 16. 19. 70.]
BPMeds [ 0.  1. nan]
totChol [195. 250. 245. 225. 285. 228. 205. 313. 260. 254. 247. 294. 332. 226.
 221. 232. 291. 190. 185. 234. 215. 270. 272. 295. 209. 175. 214. 257.
 178. 233. 180. 243. 237.  nan 311. 208. 252. 261. 179. 194. 267. 216.
 240. 266. 255. 220. 235. 212. 223. 300. 302. 248. 200. 189. 258. 202.
 213. 183. 274. 170. 210. 197. 326. 188. 256. 244. 193. 239. 296. 269.
 275. 268. 265. 173. 273. 290. 278. 264. 282. 241. 288. 222. 303. 246.
 150. 187. 286. 154. 279. 293. 259. 219. 230. 320. 312. 165. 159. 174.
 242. 301. 167. 308. 325. 229. 236. 224. 253. 464. 171. 186. 227. 249.
 176. 163. 191. 263. 196. 310. 164. 135. 238. 207. 342. 287. 182. 352.
 284. 217. 203. 262. 129. 155. 323. 206. 283. 319. 304. 340. 328. 280.
 368. 218. 276. 339. 231. 198. 177. 201. 277. 184. 199. 168. 292. 305.
 306. 15[...]<jupyter_text>Filling NAN with mode* Filled up NAN with mode, for the numerical columns which have less than 5 unique values* After this method 2 out of 7 columns are filled with mode<jupyter_code>for i in range(len(columns_with_null_values)):
    a = len(raw_data[columns_with_null_values[i]].unique())
    if a <= 5:
        raw_data[columns_with_null_values[i]].fillna(raw_data[columns_with_null_values[i]].mode()[0], inplace=True)
raw_data<jupyter_output><empty_output><jupyter_text>Filling up null values using median or mean* If the absolute difference between median and mean is low, then we fill null with mean values. However if absolute difference between median and mean is high, then median is used to fill the null rows.* After this process, there are no null values in the columns<jupyter_code>for i in range(len(columns_with_null_values)):
    a = len(raw_data[columns_with_null_values[i]].unique())
    mean_value = raw_data[columns_with_null_values[i]].mean()
    median_value = raw_data[columns_with_null_values[i]].median()
    if a > 10:
        if abs(mean_value - median_value) < 100:
#             print("fill with mean values for column: ",listed_columns_numeric_1[i] )
            raw_data[columns_with_null_values[i]] = raw_data[columns_with_null_values[i]].fillna(mean_value)
        else:
#             print("fill with median values for column: ", listed_columns_numeric_1[i])
            raw_data[columns_with_null_values[i]] = raw_data[columns_with_null_values[i]].fillna(median_value)
raw_data
print('Number of null values in each column')
print(raw_data.isnull().sum())<jupyter_output>Number of null values in each column
male               0
age                0
education          0
currentSmoker      0
cigsPerDay         0
BPMeds             0
prevalentStroke    0
prevalentHyp       0
diabetes           0
totChol            0
sysBP              0
diaBP              0
BMI                0
heartRate          0
glucose            0
TenYearCHD         0
dtype: int64<jupyter_text>Random Forest algorithm<jupyter_code>def split_for_validation(a,n):
    '''
    a = dataframe,
    n = percentage of split
    '''
    return a[: len(a) - (int(len(a) * n))].copy(), a[len(a) - (int(len(a) * n)):].copy()    

# function for random forest algorithm classifier

def rand_forest_classifier(raw_data, validation_split):
    feature_columns = raw_data.iloc[:,:-1].values
    dependent_column = raw_data.iloc[:,-1].values
    X_train, X_valid = split_for_validation(a = feature_columns,
                                            n = validation_split)
    y_train, y_valid = split_for_validation(a = dependent_column,
                                            n = validation_split)
    print('Training data: ', X_train.shape, y_train.shape)
    print('Validation data: ', X_valid.shape, y_valid.shape)

    m = RandomForestClassifier(n_jobs=-1)
    m.fit(X_train, y_train)

    predTree = m.predict(X_valid)
    print()
    print('Training accuracy: ', round((m.score(X_train, y_train))*100,2),'%')
    print()
    print('Validation accuracy: ', round((m.score(X_valid, y_valid))*100,2),'%')

    from sklearn.metrics import confusion_matrix, plot_confusion_matrix
    plot_confusion_matrix(m, X_valid, y_valid)
    plt.title('Confusion Matrix')
    plt.show()
    
    
raw_data = shuffle(raw_data)
rand_forest_classifier(raw_data = raw_data,
                       validation_split = 0.2)<jupyter_output>Training data:  (3391, 15) (3391,)
Validation data:  (847, 15) (847,)

Training accuracy:  100.0 %

Validation accuracy:  85.48 %<jupyter_text>Logistic Regression<jupyter_code>def log_regression(raw_data, validation_split):
    feature_columns = raw_data.iloc[:,:-1].values
    dependent_column = raw_data.iloc[:,-1].values
    X_train, X_valid = split_for_validation(a = feature_columns,
                                            n = validation_split)
    y_train, y_valid = split_for_validation(a = dependent_column,
                                            n = validation_split)
    print('Training data: ', X_train.shape, y_train.shape)
    print('Validation data: ', X_valid.shape, y_valid.shape)


    model = LogisticRegression()
    model.fit(X_train, y_train)
    predTree = model.predict(X_valid)
    print('Training accuracy: ', round((model.score(X_train, y_train))*100,2),'%')
    print()
    print('Validation accuracy: ', round((model.score(X_valid, y_valid))*100,2),'%')   
    
    from sklearn.metrics import confusion_matrix, plot_confusion_matrix
    
    plot_confusion_matrix(model, X_valid, y_valid)
    plt.title('Confusion Matrix')
    plt.show()
raw_data = shuffle(raw_data)
log_regression(raw_data = raw_data,
                       validation_split = 0.2)<jupyter_output>Training data:  (3391, 15) (3391,)
Validation data:  (847, 15) (847,)
Training accuracy:  84.87 %

Validation accuracy:  85.12 %
{'fraction_non_alphanumeric': 0.12381396795769171, 'fraction_numerical': 0.09861564784569918, 'mean_word_length': 2.932721712538226, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '471603', 'license': 'MIT', 'n_tokens_mistral': 2773, 'n_tokens_neox': 2167, 'n_words': 584, 'path': 'predict heart disease/predict_heart_disease.ipynb', 'repo_name': 'arjunjanamatti/machine_learning_kaggle_datasets', 'size': '6429'}