<jupyter_start><jupyter_text>import, read and introduction<jupyter_code>import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt
import random<jupyter_output><empty_output><jupyter_text>Read train and test data files<jupyter_code>df = pd.read_csv("train.csv")
test_df = pd.read_csv("test.csv")
df.shape<jupyter_output><empty_output><jupyter_text>Preprocessing  Check for missing values<jupyter_code>df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Droping columns with more than 20000 null values<jupyter_code>df.drop(['REGISTRATION_03','CARTYPE_05_CAT','CARTYPE_14','CARTYPE_03_CAT'],axis=1,inplace=True)
df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Dropping the unique identification column<jupyter_code>df = df.drop(['id'],axis=1) # removing id from the data of computation as it is not related to prediction<jupyter_output><empty_output><jupyter_text>Dropping rows with na values<jupyter_code># df = df.dropna()
df=df.fillna(method='ffill')
for i in list(df.isnull().sum()):
 if(i != 0):
   print("problem")<jupyter_output><empty_output><jupyter_text>Check for question marks<jupyter_code>question_marks = list((df == "?").sum())
for i in question_marks:
  if i !=0 :
    print("problem")<jupyter_output><empty_output><jupyter_text>Check for duplicated rows<jupyter_code>df.duplicated().sum()<jupyter_output><empty_output><jupyter_text>There are no duplicate rows.  Categorical features<jupyter_code>df.nunique()<jupyter_output><empty_output><jupyter_text>Encoding<jupyter_code># new_column = pd.cut(df.CARTYPE_13,bins=[0,4.3921793760000005,5.0899999192,5.5847490088,5.9634146304,6.3249734592,6.6978225152,7.0661572663999985,7.4562481424,7.8857293432,8.423108709600001,9.181728172,10.4332589616,100],labels=[0,1,2,3,4,5,6,7,8,9,10,11,12])
# new_column.rename({'CARTYPE_13': 'CARTYPE_13_modified'}, inplace=True)
# df = pd.concat([df,new_column], axis=1, join='inner')
# df.head()
df.drop(['METRIC_06','METRIC_11','METRIC_14','METRIC_10','CARTYPE_12'],axis=1,inplace=True)

# sus = ['CONDITION_12_BIN','CARTYPE_12','CONDITION_10_BIN','CARTYPE_11_CAT','CONDITION_13_BIN','CARTYPE_10_CAT','CONDITION_14','CONDITION_11_BIN']
# df.drop(sus,axis=1,inplace=True)
# column_names_to_one_hot = list(df.columns).remove('target')
# column_names_to_one_hot = column_names_to_one_hot.remove('CARTYPE_13')
# print(column_names_to_one_hot)

for i in df.columns:
    if i =="CARTYPE_13":
        continue
    elif i =="target":
        continue
    else:
        df[i] = df[i].astype('object')

ctype13=df["CARTYPE_13"]
final_X_test = pd.get_dummies(data=df.drop(['CARTYPE_13'],axis=1))
final_X_test["CARTYPE_13"] = ctype13

# df = pd.get_dummies(df, columns=column_names_to_one_hot)
c = final_X_test.columns<jupyter_output><empty_output><jupyter_text>Model generation  train_test_split_data<jupyter_code># test_X = df.drop(axis="columns",labels=["target"]).to_numpy().astype(np.float64)
# test_y = df['target'].to_numpy().astype(np.float64)

df = final_X_test

test_X = df.drop(axis="columns",labels=["target"]).to_numpy().astype(np.float64)
test_y = df['target'].to_numpy().astype(np.float64)

# test_X = final_X_test.to_numpy().astype(np.float64)
# test_y = y.to_numpy().astype(np.float64)

df['target'].value_counts()
idxes = df[df['target']==0].index
idxes = idxes[:245000]
df = df.drop(idxes)
df['target'].value_counts()

X = df.drop(axis="columns",labels=["target"]).to_numpy().astype(np.float64)
Y = df['target'].to_numpy().astype(np.float64)

shuffler = np.random.permutation(len(X))
train_X = X[shuffler]
train_y = Y[shuffler]<jupyter_output><empty_output><jupyter_text>Logistic regression<jupyter_code>from sklearn.linear_model import LogisticRegression 
from sklearn.metrics import accuracy_score, f1_score, roc_auc_score

lreg = LogisticRegression(solver='liblinear')
lreg.fit(train_X,train_y)<jupyter_output><empty_output><jupyter_text>testing<jupyter_code>train_yhat = lreg.predict_proba(train_X)
test_yhat = lreg.predict_proba(test_X)

print("training score = ", roc_auc_score(train_y, train_yhat[:,1]))
print("test score = ", roc_auc_score(test_y,test_yhat[:,1]))

test_df=test_df.fillna(method='ffill')
test_df.drop(['REGISTRATION_03','CARTYPE_05_CAT','CARTYPE_14','CARTYPE_03_CAT'],axis=1,inplace=True)
# sus = ['CONDITION_12_BIN','CARTYPE_12','CONDITION_10_BIN','CARTYPE_11_CAT','CONDITION_13_BIN','CARTYPE_10_CAT','CONDITION_14','CONDITION_11_BIN']
# test_df.drop(sus,axis=1,inplace=True)
# new_column = pd.cut(test_df.CARTYPE_13,bins=[0,4.3921793760000005,5.0899999192,5.5847490088,5.9634146304,6.3249734592,6.6978225152,7.0661572663999985,7.4562481424,7.8857293432,8.423108709600001,9.181728172,10.4332589616,100],labels=[0,1,2,3,4,5,6,7,8,9,10,11,12])
# new_column.rename({'CARTYPE_13': 'CARTYPE_13_modified'}, inplace=True)
# test_df = pd.concat([test_df,new_column], axis=1, join='inner')
# test_df.head()
test_df.drop(['METRIC_06','METRIC_11','METRIC_14','METRIC_10','CARTYPE_12'],axis=1,inplace=True)
l = list(test_df.columns.values).remove("id")
for i in test_df.columns:
    if i =="CARTYPE_13":
        continue
    elif i =="id":
        continue
    else:
        test_df[i] = test_df[i].astype('object')

ctype13=test_df["CARTYPE_13"]
ID_valid=test_df["id"]
final_X_test = pd.get_dummies(data=test_df.drop(['CARTYPE_13','id'],axis=1))
final_X_test["CARTYPE_13"] = ctype13
final_X_test["id"] = ID_valid

test_df.isnull().sum()
test_X = final_X_test.drop(axis="columns", labels=['id']).to_numpy().astype(np.float64)
c1 = final_X_test.columns

for i in c:
    if i in c1:
        continue
    else:
        print(i)
test_yhat = lreg.predict_proba(test_X)
y_pred_df = pd.DataFrame(data=test_yhat[:,1], columns = ["target"])

submission_df = pd.concat([test_df["id"], y_pred_df["target"]], axis=1, join='inner')
submission_df.to_csv("submission.csv", index = False)
print(submission_df.shape)<jupyter_output>(297606, 2)
{'fraction_non_alphanumeric': 0.1658727458319156, 'fraction_numerical': 0.08523307247363049, 'mean_word_length': 8.831103678929766, 'pattern_counts': {'":': 4, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '517145', 'license': 'MIT', 'n_tokens_mistral': 2697, 'n_tokens_neox': 2429, 'n_words': 268, 'path': 'ML_hackathon_classification_best.ipynb', 'repo_name': 'architsangal/Machine-Learning-Hackathon-1-Classification', 'size': '5878'}