<jupyter_start><jupyter_text>Big Data -- Sample Data!<jupyter_code>from sklearn.feature_extraction import DictVectorizer
import pandas as pd
import numpy as np
train_sample = pd.read_csv('/home/vincent/Desktop/datasets/' + 'test.csv', nrows=1000)
labels = pd.read_csv('/home/vincent/Desktop/datasets/' + 'trainLabels.csv', nrows=1000)
train_sample.head()
labels.head()
labels.columns
train_sample[['id']] = range(1, 1001)<jupyter_output><empty_output><jupyter_text>Try to make something useful<jupyter_code>train = pd.merge(train_sample, labels, on = 'id')
train.head()
from collections import Counter
Counter([name[0] for name in train.columns])
train.columns<jupyter_output><empty_output><jupyter_text>Categorical values encoding<jupyter_code>train.replace('YES', 1, inplace = True)
train.replace('NO', 0, inplace = True)
train.replace('nan', np.NaN, inplace = True)
train.head()
names_categorical = []
X_numerical = []

for name in train.columns:
    if name.startswith('x'):
        # 找出每个列中最多的数据类型，因为每一列中可能会有str或数值类型的值
        column_type, _ = max(Counter(map(lambda x: str(type(x)), train[name])).items(), key = lambda x: x[1])
        if column_type == str(str):
            train[name] = train[name].apply(str)
            names_categorical.append(name)
        else:
            X_numerical.append(train[name].fillna(-999))
vec = DictVectorizer()
X_sparse = vec.fit_transform(train[names_categorical].T.to_dict().values())
X_numerical = np.column_stack(X_numerical)
X_numerical = np.nan_to_num(X_numerical)
print(X_numerical.shape)
print(X_sparse.shape)
from sklearn.metrics import roc_auc_score, f1_score, log_loss, make_scorer
from sklearn.svm import LinearSVC
from sklearn.cross_validation import cross_val_score, train_test_split
from sklearn.ensemble import RandomForestClassifier
y_columns = [name for name in train.columns if name.startswith('y')]
X_numerical_base, X_numerical_meta, X_sparse_base, X_sparse_meta, y_base, y_meta = train_test_split(
    X_numerical, 
    X_sparse, 
    train[y_columns].values,
    test_size = 0.5
)
X_meta = []
print('start training')
for i in range(y_base.shape[1]):
    y = y_base[:, i]
    if len(np.unique(y)) == 2:
        rf = RandomForestClassifier(n_estimators = 10, n_jobs = -1)
        rf.fit(X_numerical_base, y)
        X_meta.append(rf.predict_proba(X_numerical_meta))
        
        svm = LinearSVC()
        svm.fit(X_sparse_base, y)
        X_meta.append(svm.decision_function(X_sparse_meta))
print('end training')
X_meta = np.column_stack(X_meta)
print(X_meta.shape)<jupyter_output>(500, 75)
{'fraction_non_alphanumeric': 0.12734375, 'fraction_numerical': 0.012890625, 'mean_word_length': 4.983644859813084, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '162522', 'license': 'Apache-2.0', 'n_tokens_mistral': 978, 'n_tokens_neox': 943, 'n_words': 170, 'path': 'comprehensive/Tradeshift Text Classification.ipynb', 'repo_name': 'KEVINYZY/jupyter_hub', 'size': '2560'}