<jupyter_start><jupyter_text>Import useful stuff and define ancillary functions<jupyter_code>%pylab inline
%load_ext autoreload
%autoreload 2

from __future__ import division

from collections import defaultdict, namedtuple
import cPickle as pickle
from datetime import datetime, timedelta
from functools import partial
import inspect
import json
import os
import re
import sys

import numpy as np
import pandas as pd
import seaborn as sn
import sklearn as sl<jupyter_output>Populating the interactive namespace from numpy and matplotlib<jupyter_text>Load train dataUsing pandas' read_csv with all the defaults<jupyter_code>if os.name == 'nt':
    TRAIN_PATH = r'D:\train.csv'
    PTRAIN_PATH = r'D:\train_preprocessed_all.csv'
    TEST_PATH = r'D:\test.csv'
    GOOGNEWS_PATH = r'D:\GoogleNews-vectors-negative300.bin.gz'
    VOCAB_PATH = r'D:\big.txt'
else:
    TRAIN_PATH = r'/media/mtambos/speedy/train.csv'
    PTRAIN_PATH = r'/media/mtambos/speedy/train_preprocessed_all.csv'
    TEST_PATH = r'/media/mtambos/speedy/test.csv'
    GOOGNEWS_PATH = r'/media/mtambos/speedy/GoogleNews-vectors-negative300.bin.gz'
    VOCAB_PATH = r'/media/mtambos/speedy/big.txt'
#df_orig = pd.read_csv(TRAIN_PATH, index_col="ID")
df = pd.read_csv(PTRAIN_PATH, index_col="ID")
#df<jupyter_output>/home/mtambos/anaconda/lib/python2.7/site-packages/pandas/io/parsers.py:1170: DtypeWarning: Columns (8,9,10,11,12,23,158,174,177,178,180,184,187) have mixed types. Specify dtype option on import or set low_memory=False.
  data = self._reader.read(nrows)<jupyter_text>Define columns<jupyter_code>bool_cols = ['VAR_0008', 'VAR_0009', 'VAR_0010',
             'VAR_0011', 'VAR_0012', 'VAR_0043',
             'VAR_0196', 'VAR_0226', 'VAR_0229',
             'VAR_0230', 'VAR_0232', 'VAR_0236',
             'VAR_0239']
bool_cols += [c for c in df.columns if df[c].dtype == bool]
bool_cols = list(set(bool_cols))

int_cols = ['VAR_0013', 'VAR_0045', 'VAR_0198',
            'VAR_0227', 'VAR_0231', 'VAR_0233',
            'VAR_0237', 'VAR_0241']
int_cols += [c for c in df.columns if df[c].dtype == int]
int_cols = list(set(int_cols))<jupyter_output><empty_output><jupyter_text>See if the classes are skewed<jupyter_code>neg_samples_count = len(df['target'][df['target']==0])
pos_samples_count = len(df['target'][df['target']==1])
print '%s negative samples; %.2f%% of total' % (neg_samples_count, neg_samples_count/len(df)*100)
print '%s positive samples; %.2f%% of total' % (pos_samples_count, pos_samples_count/len(df)*100)<jupyter_output>111458 negative samples; 76.75% of total
33773 positive samples; 23.25% of total<jupyter_text>Cast bool columns as bool<jupyter_code>def clean_bools(bool_val):
    if isinstance(bool_val, bool):
        return bool_val
    try:
        return bool(bool_val)
    except:
        return np.nan

df[bool_cols] = df[bool_cols].applymap(clean_bools)
df[bool_cols] = df[bool_cols].astype(np.bool)
df[bool_cols]<jupyter_output><empty_output><jupyter_text>Cast int columns as int<jupyter_code>def clean_ints(int_val):
    if isinstance(int_val, int):
        return int_val
    try:
        return int(int_val)
    except:
        return np.nan

df[int_cols] = df[int_cols].applymap(clean_ints)
df[int_cols] = df[int_cols].astype(np.int)
df[int_cols]<jupyter_output><empty_output><jupyter_text>Remove columns with low standard deviation<jupyter_code>X = df.std(skipna=True)
X.sort()
zero_std_cols = X[X == 0].index.tolist()
zero_std_cols<jupyter_output><empty_output><jupyter_text>Drop duplicated rows and columns   Drop duplicate rows<jupyter_code>df.drop_duplicates(inplace=True)<jupyter_output><empty_output><jupyter_text>Drop duplicate bool columns<jupyter_code>cols_to_drop = set()
for i in range(len(bool_cols) - 1):
    c1 = bool_cols[i]
    print "Checking duplicates of column " + c1
    for j in range(i+1, len(bool_cols)):
        c2 = bool_cols[j]
        if (df[c1] == df[c2]).all():
            cols_to_drop.add(c2)
    print "Duplicates found: " + str(cols_to_drop)
df = df.drop(cols_to_drop, axis=1)
for c in cols_to_drop:
    bool_cols.remove(c)<jupyter_output><empty_output><jupyter_text>Drop duplicate int columns<jupyter_code>cols_to_drop = set()
for i in range(len(int_cols) - 1):
    c1 = int_cols[i]
    for j in range(i+1, len(int_cols)):
        c2 = int_cols[j]
        if (df[c1] == df[c2]).all():
            cols_to_drop.add(c2)
print "Duplicates found: " + str(cols_to_drop)<jupyter_output>Duplicates found: set([])<jupyter_text>Impute missing bool and int values<jupyter_code>nan_cols = df[bool_cols + int_cols].isnull().any()
nan_cols[nan_cols]<jupyter_output><empty_output><jupyter_text>So, no missing values to impute.  Remove bool and int columns with only NaNs<jupyter_code>nan_cols = df[bool_cols + int_cols].isnull().all()
nan_cols[nan_cols]<jupyter_output><empty_output><jupyter_text>Save preprocessed data to another csv file<jupyter_code>df.to_csv(PTRAIN_PATH)
with open('int_cols.pickle', 'wb') as fp:
    pickle.dump(int_cols, fp)
with open('bool_cols.pickle', 'wb') as fp:
    pickle.dump(bool_cols, fp)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12755503144654087, 'fraction_numerical': 0.03419811320754717, 'mean_word_length': 5.198538367844092, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '562015', 'license': 'MIT', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1921, 'n_words': 375, 'path': 'Springleaf - preprocess.ipynb', 'repo_name': 'mtambos/springleaf', 'size': '5088'}