<jupyter_start><jupyter_text>Visualize data<jupyter_code>train.head()
train.count()
train.describe()
train.dtypes
test.count()
remove_std = []
for col in train.columns:
    if train[col].std() == 0:
        remove_std.append(col)
        
train.drop(remove_std, axis = 1, inplace= True)
test.drop(remove_std, axis = 1, inplace = True)
from pandas.core.common import array_equivalent

def duplicate_columns(frame):
    groups = frame.columns.to_series().groupby(frame.dtypes).groups
    dups = []

    for t, v in groups.items():

        cs = frame[v].columns
        vs = frame[v]
        lcs = len(cs)

        for i in range(lcs):
            ia = vs.iloc[:,i].values
            for j in range(i+1, lcs):
                ja = vs.iloc[:,j].values
                if array_equivalent(ia, ja):
                    dups.append(cs[i])
                    break

    return dups
mms = MinMaxScaler()
train_new = mms.fit_transform(train)

train_new = train_new.astype('float32')
train_new = pd.DataFrame(train_new, columns=train.columns)

remove_same = duplicate_columns(train_new)
train.drop(remove_same, axis = 1, inplace= True)
test.drop(remove_same, axis = 1, inplace = True)
# set data for train-test
data_x = train.ix[:, (train.columns != 'TARGET') & (train.columns != 'ID')].values
data_y = train.TARGET.values

data_x_test = test.ix[:, test.columns != 'ID'].values
# set model
clf1 = RandomForestClassifier(n_jobs=3, n_estimators=250, oob_score=True)
cccv1 = CalibratedClassifierCV(clf1, method = 'isotonic', cv = 5)
# rmf_cv with 100 trees
cross_val_score(cccv1, data_x, data_y, cv = 3, scoring='roc_auc')
# rmf with 250 trees
cross_val_score(cccv1, data_x, data_y, cv = 3, scoring='roc_auc')
# train and predict
cccv1.fit(data_x, data_y);
pred = cccv1.predict_proba(data_x_test)
# make submission
sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
sub.TARGET = pred[:, 1]
sub.to_csv(os.path.join(subs_dir, 'sub02.csv'), index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13839959738298943, 'fraction_numerical': 0.01560140915953699, 'mean_word_length': 3.7109004739336493, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '178735', 'license': 'MIT', 'n_tokens_mistral': 750, 'n_tokens_neox': 719, 'n_words': 153, 'path': 'trials/trial1.ipynb', 'repo_name': 'faizankshaikh/santander_kaggle', 'size': '1987'}