<jupyter_start><jupyter_text>01 - TSFresh and Data Analysis  Imports<jupyter_code>import numpy as np
import pandas as pd
import json 
from functools import reduce
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.feature_extraction import DictVectorizer
from sklearn.externals import joblib
import pickle
import os.path
from sklearn.model_selection import KFold
from sklearn.decomposition import PCA, IncrementalPCA, SparsePCA, TruncatedSVD
import scipy.sparse
from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler
import catboost
from catboost import CatBoostClassifier, Pool
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

sns.set(style="white")<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>nrows = None
rare_key_threshold = 10
nfolds = 100
data_folder = 'data/'
train_folds_folder = "folds_train/"
test_folds_folder = "folds_test/"
model_folder = "models/"

target_name = 'target'
test_index_name = 'tst_index'
target_fn = model_folder +'target.pkl'

label_encoder_fn = model_folder + "label_encoder.pkl"
weight_multiplier_fn = model_folder + "weight_multiplier.pkl"
json_metas_fn = model_folder + "json_metas.pkl"
train_indexes_fn = model_folder + "train_indexes.pkl"
test_indexes_fn = model_folder + "test_indexes.pkl"
from datetime import datetime
today = datetime.today()<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def stayOnlyKeys(keys, dictionary):
    for key in list(dictionary.keys()):
        if key not in keys:
            del dictionary[key]
    return dictionary
def LoadTarget():
    dft = pd.read_csv(data_folder+'mlboot_train_answers.tsv',
                      delimiter='\t',
                      encoding='utf-8',
                      header=0)
    dft.columns = ['cuid','target']
    dft = dft.set_index('cuid')
    
    return dft
def LoadTest():
    dftst = pd.read_csv(data_folder+'mlboot_test.tsv',
                          delimiter='\t',
                          encoding='utf-8',
                          header=0)
    dftst[test_index_name] = dftst.index+2
    dftst = dftst.set_index('cuid')
    
    return dftst
def LoadData(nrows = None):
    df = pd.read_csv(data_folder+'mlboot_data.tsv',
                delimiter='\t',
                encoding='utf-8',
                nrows=nrows,
                header=None,
                names=['cuid','cat_feature', 'json1', 'json2', 'json3', 'dt_diff'],
                index_col='cuid'
                )
    
    return df
def LoadRawDataSet(nrows):
    df = LoadData(nrows)
    dftst = LoadTest()
    dft = LoadTarget()
    
    df = df.join(dft)
    df = df.join(dftst)
    
    return df
def GetTrainDataSet(df):
    print('Rows in input:',len(df.index))
    print('Rows in output:',len(df[df[target_name].notnull()].index))
    
    return df[df[target_name].notnull()]
def GetTestDataSet(df):
    print('Rows in input:',len(df.index))
    print('Rows in output:',len(df[df[test_index_name].notnull()].index))
    
    return df[df[test_index_name].notnull()]
def Save(obj, filename):
    joblib.dump(obj, filename)
def Load(filename):
    if os.path.isfile(filename):
        return joblib.load(filename)<jupyter_output><empty_output><jupyter_text>Loading data  From tsv files<jupyter_code>df = LoadRawDataSet(nrows)
from sklearn import preprocessing

encoder = preprocessing.LabelEncoder()
encoder.fit(df.index.unique())

Save(encoder, label_encoder_fn )
df["id"] = encoder.transform(df.index)
df.index = df["id"]
%%time
from tsfresh.feature_extraction import extract_features, EfficientFCParameters, ComprehensiveFCParameters, MinimalFCParameters
settings = MinimalFCParameters()

extracted_features = extract_features(df,
                 column_id="id",
#                  column_sort="dt_diff",
                 column_kind=None,
                 column_value="cat_feature",
                 default_fc_parameters=settings)
df = df.join(extracted_features)
dft = GetTrainDataSet(df)
dft = dft.drop([test_index_name], axis=1)
dftst = GetTestDataSet(df)
dftst = dftst.drop([target_name], axis=1)
target_df = dft.groupby("id", as_index=True).agg({
         'target':['max']
    })
target = target_df.values
print("Target size:",len(target))

Save(target, target_fn)
classes,counts = np.unique(target, return_counts=True)
weight_multiplier = counts[0]/counts[1]
print('Weight multiplier for imbalanced dataset:',weight_multiplier)

Save(weight_multiplier, weight_multiplier_fn)<jupyter_output>Weight multiplier for imbalanced dataset: 18.951239977624464<jupyter_text>Json counters research Нужно, чтобы ключи счётчиков в трейне и тесте пересекались.Не имеет смысла тренировать модель на тех счётчкиках, которые в тестовых данных не встречаются.Аналогично с тестовыми данными - из них нужно убрать те счётчики, которые не встречаются в трейне, так как они всё равно не помогут модели<jupyter_code>def analyze_keys(text, meta, dataset_type="train"):
    dictionary = json.loads(text)
    keys = dictionary.keys()
    
    meta[dataset_type].update(keys)
    
    for key in keys:
        if key in meta["key_counter"]:
            meta["key_counter"][key] = meta["key_counter"][key]+1
        else:
            meta["key_counter"][key] = 1
        
        if key in meta["valued_key_counter"] and dictionary[key] is not None and dictionary[key] > 0:
                meta["valued_key_counter"][key] = meta["valued_key_counter"][key]+1
        if key not in meta["valued_key_counter"] and dictionary[key] is not None and dictionary[key] > 0:
                meta["valued_key_counter"][key] = 1
            
            
def GetNonTrashKeys(dic):
    global rare_key_threshold
    
    non_trash_keys_dict = dict( (key, value) for (key, value) in dic.items() if value > rare_key_threshold )
    non_trash_keys = set(list(non_trash_keys_dict.keys()))
    
    return non_trash_keys
def jobs_manager():
    from IPython.lib.backgroundjobs import BackgroundJobManager
    from IPython.core.magic import register_line_magic
    from IPython import get_ipython

    jobs = BackgroundJobManager()

    @register_line_magic
    def job(line):
        ip = get_ipython()
        jobs.new(line, ip.user_global_ns)

    return jobs

def get_chunks(sequence, count):
    count = min(count, len(sequence))
    chunks = [[] for _ in range(count)]
    for index, item in enumerate(sequence):
        chunks[index % count].append(item) 
    return chunks

def log_progress(sequence, every=None, size=None):
    from ipywidgets import IntProgress, HTML, VBox
    from IPython.display import display

    is_iterator = False
    if size is None:
        try:
            size = len(sequence)
        except TypeError:
            is_iterator = True
    if size is not None:
        if every is None:
            if size <= 200:
                every = 1
            else:
                every = size / 200     # every 0.5%
    else:
        assert every is not None, 'sequence is iterator, set every'

    if is_iterator:
        progress = IntProgress(min=0, max=1, value=1)
        progress.bar_style = 'info'
    else:
        progress = IntProgress(min=0, max=size, value=0)
    label = HTML()
    box = VBox(children=[label, progress])
    display(box)

    index = 0
    try:
        for index, record in enumerate(sequence, 1):
            if index == 1 or index % every == 0:
                if is_iterator:
                    label.value = '{index} / ?'.format(index=index)
                else:
                    progress.value = index
                    label.value = u'{index} / {size}'.format(
                        index=index,
                        size=size
                    )
            yield record
    except:
        progress.bar_style = 'danger'
        raise
    else:
        progress.bar_style = 'success'
        progress.value = index
        label.value = str(index or '?')

jobs = jobs_manager()
def kill_thread(thread):
    import ctypes
    
    id = thread.ident
    code = ctypes.pythonapi.PyThreadState_SetAsyncExc(
        ctypes.c_long(id),
        ctypes.py_object(SystemError)
    )
    if code == 0:
        raise ValueError('invalid thread id')
    elif code != 1:
        ctypes.pythonapi.PyThreadState_SetAsyncExc(
            ctypes.c_long(id),
            ctypes.c_long(0)
        )
        raise SystemError('PyThreadState_SetAsyncExc failed')
json_metas = []

def build_json_meta(i):
    global json_metas
    global dft
    global dftst
    
    meta = {
        "train":set([]),
        "test":set([]),
        "total":set([]),
        "cross":set([]),
        "key_counter":{},
        "valued_key_counter":{},
        "non_trash":set([]),
        "important_keys":set([]),
        "i":i,
        "column":"json"+str(i)
    }


    print("Counting keys:",i)
    dft[meta["column"]].apply(lambda text: analyze_keys(text, meta, "train"))
    dftst[meta["column"]].apply(lambda text: analyze_keys(text, meta, "test"))
    
    meta["total"] = meta["train"].copy()
    meta["total"].update(meta["test"])
    
    meta["cross"] = meta["train"] & meta["test"]

    meta["non_trash"] = GetNonTrashKeys(meta["key_counter"])
    meta["important_keys"] = meta["non_trash"] & meta["cross"]

    print("DictVectorizer fiting:",i)
    dict_vector = dict.fromkeys(meta["important_keys"], 1)

    vectorizer = DictVectorizer(sparse=True).fit([dict_vector])

    meta["vectorizer"] = vectorizer
    
    json_metas.append(meta)
%%time
for chunk in get_chunks(range(1,4), 3):
    %job [build_json_meta(index) for index in log_progress(chunk, every=1)]    
    
for thread in jobs.running:
    thread.join()
for i in range(0,3):
    json_metas[i]["valued_non_trash"] = GetNonTrashKeys(json_metas[i]["valued_key_counter"])
for i in range(0,3):
    print("non_trash",len(json_metas[i]["non_trash"]))
    print("valued_non_trash",len(json_metas[i]["valued_non_trash"]))
    print("important_keys",len(json_metas[i]["important_keys"]))    
%%time
Save(json_metas, json_metas_fn)<jupyter_output>CPU times: user 49.8 s, sys: 800 ms, total: 50.6 s
Wall time: 50.6 s<jupyter_text>Spliting on folds<jupyter_code>train_indexes = np.array_split(dft.index.unique(), nfolds)

dft = dft.drop([target_name], axis=1)
i = 0

for index in train_indexes:
    dfi = dft[dft.index.isin(index)]
    
    dfi_fn = train_folds_folder+"dfi{!s}.pkl".format(i)
    
    Save(dfi, dfi_fn)
    i = i+1
    
Save(train_indexes, train_indexes_fn)
dftst = dftst.sort_values(test_index_name)
dftst = dftst.drop([test_index_name], axis=1)
test_indexes = np.array_split(dftst.index.unique(), nfolds)
dftst.head(10)
i = 0

for index in test_indexes:
    dfi = dftst[dftst.index.isin(index)]
    
    dfi_fn = test_folds_folder+"dfi{!s}.pkl".format(i)
    
    Save(dfi, dfi_fn)
    i = i+1

Save(test_indexes, test_indexes_fn)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11722421896599391, 'fraction_numerical': 0.008570638650815593, 'mean_word_length': 3.1419847328244277, 'pattern_counts': {'":': 10, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '435960', 'license': 'MIT', 'n_tokens_mistral': 3731, 'n_tokens_neox': 3668, 'n_words': 748, 'path': '01 - TsFresh and Data Analysis.ipynb', 'repo_name': 'avdeev-andrew/mlbootcamp6', 'size': '10851'}