<jupyter_start><jupyter_text>Instruction To obtain the dataset Kick used for algorithms comparison:1) Download `training.csv` file from "Don't Get Kicked!" competition on Kaggle: https://www.kaggle.com/c/DontGetKicked/data (you can download .zip archive and extract the file from it).2) Put it to the same directory as this notebook.3) Run all the cells of this notebook successively to produce files for training and testing.<jupyter_code>resulting_train_filename = "train"
resulting_test_filename = "test"<jupyter_output><empty_output><jupyter_text>Preparing the data<jupyter_code>import pandas as pd
import re
data = pd.read_csv("training.csv")
data.head()
target = data["IsBadBuy"].apply(lambda x: 1.0 if x == 0 else -1.0)
data["PurchYear"] = pd.DatetimeIndex(data['PurchDate']).year
data["PurchMonth"] = pd.DatetimeIndex(data['PurchDate']).month
data["PurchDay"] = pd.DatetimeIndex(data['PurchDate']).day
data["PurchWeekday"] = pd.DatetimeIndex(data['PurchDate']).weekday
data.drop(["RefId", "IsBadBuy", "PurchDate"], axis=1, inplace=True)
data.shape<jupyter_output><empty_output><jupyter_text>Preparing categorical features<jupyter_code>categorical_features = set([0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 23, 24, 25, 26, 27, 29, 31, 32, 33, 34])
def clean_string(s):
    return re.sub('[^A-Za-z0-9]+', "_", str(s))

for i in categorical_features:
    data[data.columns[i]] = data[data.columns[i]].apply(clean_string)<jupyter_output><empty_output><jupyter_text>Preparing numerical features<jupyter_code>columns_to_impute = []
for i, column in enumerate(data.columns):
    if i not in categorical_features and pd.isnull(data[column]).any():
        columns_to_impute.append(column)
columns_to_impute
for column_name in columns_to_impute:
    data[column_name + "_imputed"] = pd.isnull(data[column_name]).astype(float)
    data[column_name].fillna(0, inplace=True)
for i, column in enumerate(data.columns):
    if i not in categorical_features:
        data[column] = data[column].astype(float)
data.shape<jupyter_output><empty_output><jupyter_text>Preparing train/test split<jupyter_code>train_idx = pd.read_csv("stratified_train_idx.txt", header=None)
test_idx = pd.read_csv("stratified_test_idx.txt", header=None)
Xtrain = data.iloc[train_idx[0]]
Ytrain = target.iloc[train_idx[0]]
Xtest = data.iloc[test_idx[0]]
Ytest = target.iloc[test_idx[0]]
# creating file with features
def prepare_pool(data, labels, filename):
    X = data.values
    y = labels.values
    with open(filename, "w") as fout:
        for i in range(data.shape[0]):
            fout.write(str(y[i]) + "\t" + "\t".join(map(str, X[i])) + "\n")
prepare_pool(Xtrain, Ytrain, resulting_train_filename)
prepare_pool(Xtest, Ytest, resulting_test_filename)
with open(resulting_train_filename + '.cd', 'w') as fout:
    fout.write('0\tTarget\n')
    for cat_f_id in sorted(categorical_features):
        fout.write('{}\tCateg\n'.format(cat_f_id + 1))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14266350389698407, 'fraction_numerical': 0.018976618095560827, 'mean_word_length': 6.079136690647482, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '139729', 'license': 'Apache-2.0', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1049, 'n_words': 226, 'path': 'catboost/benchmarks/quality_benchmarks/prepare_kick/prepare_kick.ipynb', 'repo_name': 'infected-mushroom/catboost', 'size': '2951'}