<jupyter_start><jupyter_text>choose an appropriate sampler and sampling ratio by the submission<jupyter_code>def get_X_y(dataset):
    X = dataset.drop(columns=["user_id", "label"]).fillna(-1).values
    y = dataset.label.values
    return X, y


def sampler(name, ratio, random_state=0, return_indices=True, **kwargs):
    if name == "rus":
        sampler = RandomUnderSampler(
            ratio=ratio,
            return_indices=return_indices,
            random_state=random_state,
            **kwargs,
        )
    elif name == "nm":
        sampler = NearMiss(
            ratio=ratio,
            return_indices=return_indices,
            random_state=random_state,
            **kwargs,
        )
    elif name == "enn":
        sampler = EditedNearestNeighbours(
            return_indices=return_indices, random_state=random_state, **kwargs
        )
    elif name == "renn":
        sampler = RepeatedEditedNearestNeighbours(
            return_indices=return_indices, random_state=random_state, **kwargs
        )
    elif name == "allknn":
        sampler = AllKNN(
            return_indices=return_indices, random_state=random_state, **kwargs
        )
    elif name == "tl":
        sampler = TomekLinks(
            return_indices=return_indices, random_state=random_state, **kwargs
        )
    else:
        raise ValueError
    return sampler


def merge_scoring_metrics(scores, scorer):
    df = pd.DataFrame(scores)
    custom_metrics = scorer.get(filter=None)
    for metric, scores in custom_metrics.items():
        df["test_{0}".format(metric)] = scores[::2]
        df["train_{0}".format(metric)] = scores[1::2]
    return df


def score_whole_dataset(clf, dataset, pre_train=True):
    if not ("label" in dataset):
        raise ValueError("dataset must include the label column")
    X, y = get_X_y(dataset)
    if not pre_train:
        clf.fit(X, y)
    scoring, scorer = get_jdata_scoring(dataset)
    scoring["custom_index"](clf, X, y, np.arange(X.shape[0]))

    metrics = {}
    for k, v in scorer.get(filter=None).items():
        metrics["test_{}".format(k)] = v

    return pd.DataFrame(metrics)
# load training dataset
frw = FeatherReadWriter()
train = frw.read(dir="processed", name=frw.extend_name("all_merged_1.0"), nthreads=4)
label = frw.read(
    dir="processed", name=frw.extend_name("all_merged_1.0.label"), nthreads=4
)
train[label.columns] = label
X, y = get_X_y(train)

# load online dataset for submission
online_train = frw.read(dir="processed", name=frw.extend_name("all_merged_online"), nthreads=4)
online_label = frw.read(
    dir="processed", name=frw.extend_name("all_merged_online.label"), nthreads=4
)
online_train[online_label.columns] = online_label
sampling_paras = [
    ("rus", 0.1),
    ("rus", 0.01),
    ("rus", 0.001),
    ("nm", 0.1),
    ("nm", 0.01),
    ("nm", 0.001),
    ("tl", None),
    ("enn", None),
    ("renn", None),
    ("allknn", None),
]

fpath = str(PROJECT_DIR.joinpath("reports/metrics_by_samplers.csv"))
whole_dataset_metrics = pd.DataFrame()

for method, ratio in sampling_paras:
    with timer(f"method: {method}, ratio: {ratio}"):
        sampler_setting = {"name": method, "ratio": ratio, "n_jobs": 4}
        s = sampler(**sampler_setting)
        res_X, res_y, indices = s.fit_sample(X, y)
    print("Distribution of class labels after resampling {}".format(Counter(res_y)))

    clf = XGBClassifier(nthread=-1)
    with timer("start training"):
        clf.fit(res_X, res_y, verbose=3)

    score_df = score_whole_dataset(clf, online_train)
    score_df = score_df.set_index([["{0}-{1}".format(method, ratio)]])
    whole_dataset_metrics = pd.concat([whole_dataset_metrics, score_df])
    whole_dataset_metrics.to_csv(fpath)
    frw.write(
        pd.DataFrame({"index": indices}),
        "processed",
        frw.extend_name(f"all_merged_1.0_{method}_{ratio}_indices"),
    )<jupyter_output><empty_output><jupyter_text>```Distribution of class labels after resampling Counter({0: 21240, 1: 2124})method: rus, ratio: 0.1: 18.471 secDistribution of class labels after resampling Counter({0: 212400, 1: 2124})method: rus, ratio: 0.01: 39.256 secDistribution of class labels after resampling Counter({0: 2124000, 1: 2124})method: rus, ratio: 0.001: 654.605 secDistribution of class labels after resampling Counter({0: 21240, 1: 2124})method: nm, ratio: 0.1: 866.883 secDistribution of class labels after resampling Counter({0: 212400, 1: 2124})method: nm, ratio: 0.01: 798.771 secDistribution of class labels after resampling Counter({0: 2124000, 1: 2124})method: nm, ratio: 0.001: 1384.717 sec```we cannot get the results of ("tl", None), ("renn", None), ("allknn", None), ("enn", None)TODO: maybe we should standardize our datasets to speed up these processesfinally, we choose random under sampling with ratio 0.01.<jupyter_code>indices = frw.read("processed", frw.extend_name(f"all_merged_1.0_rus_0.01_indices"))
res_train = train.iloc[indices['index'], :]
res_X, res_y = get_X_y(res_train)
Counter(res_y)<jupyter_output><empty_output><jupyter_text>use cross validation to compare metrics<jupyter_code>kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=41)<jupyter_output><empty_output><jupyter_text>sklearn default metrics<jupyter_code>clf = XGBClassifier(nthread=-1)
scoring = {
    "precision": "precision",
    "recall": "recall",
    "f1": "f1",
    "neg_log_loss": "neg_log_loss",
    "roc_auc": "roc_auc",
}
scores = cross_validate(clf, res_X, res_y, cv=kfold, scoring=scoring, return_train_score=True, verbose=1)
pd.DataFrame(scores)<jupyter_output><empty_output><jupyter_text>The difference between JData Fscore and sklearn metrics<jupyter_code>clf = XGBClassifier(nthread=-1)
scoring, scorer = get_jdata_scoring(res_train)
%time scores = cross_validate(clf, res_X, res_y, cv=kfold, scoring=scoring, return_train_score=True, verbose=1)
pd.DataFrame(scores)[['test_custom_index', 'train_custom_index']]
# test metrics
pd.DataFrame(scorer.get())
merge_scoring_metrics(scores, scorer)<jupyter_output><empty_output><jupyter_text>find best estimator by gridsearchcv and use custom jdata score function<jupyter_code>scoring, scorer = get_jdata_scoring(res_train)
scoring = {'custom': scoring["custom_index"]}
refit = 'custom'
clf = XGBClassifier(nthread=-1)
n_estimators = range(50, 400, 50)
param_grid = dict(n_estimators=n_estimators)

grid_search = GridSearchCV(clf, param_grid, scoring=scoring, cv=kfold, refit=refit, return_train_score=True)
%time grid_result = grid_search.fit(res_X, res_y)

# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_["mean_test_custom"]
stds = grid_result.cv_results_["std_test_custom"]
params = grid_result.cv_results_["params"]
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))

# Plot
pyplot.errorbar(n_estimators, means, yerr=stds)
pyplot.title("XGBoost n_estimators vs JData score")
pyplot.xlabel('n_estimators')
pyplot.ylabel('JData score')
# pd.DataFrame(grid_result.cv_results_).to_csv('../reports/search_n_estimators_all_merged_1.0_rus_0.01.csv', index=False)
pd.DataFrame(grid_result.cv_results_)
scoring, scorer = get_jdata_scoring(res_train)
scoring = {'custom': scoring["custom_index"]}
refit = 'custom'

clf = XGBClassifier(nthread=-1)
max_depth = range(1, 11, 2)
print(max_depth)
param_grid = dict(max_depth=max_depth)

grid_search = GridSearchCV(clf, param_grid, scoring=scoring, cv=kfold, refit=refit, return_train_score=True)
%time grid_result = grid_search.fit(res_X, res_y)

# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_["mean_test_custom"]
stds = grid_result.cv_results_["std_test_custom"]
params = grid_result.cv_results_["params"]
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))

# plot
pyplot.errorbar(max_depth, means, yerr=stds)
pyplot.title("XGBoost max_depth vs JData score")
pyplot.xlabel('max_depth')
pyplot.ylabel('JData score')
# pd.DataFrame(grid_result.cv_results_).to_csv('../reports/search_max_depth_all_merged_1.0_rus_0.01.csv', index=False)
pd.DataFrame(grid_result.cv_results_)
scoring, scorer = get_jdata_scoring(res_train)
scoring = {'custom': scoring["custom_index"]}
refit = 'custom'

clf = XGBClassifier(nthread=-1)
n_estimators = range(150, 400, 50)
max_depth = range(3, 9, 2)
param_grid = dict(max_depth=max_depth, n_estimators=n_estimators)


grid_search = GridSearchCV(clf, param_grid, scoring=scoring, cv=kfold, refit=refit, verbose=1, return_train_score=True)
%time grid_result = grid_search.fit(res_X, res_y)

# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_["mean_test_custom"]
stds = grid_result.cv_results_["std_test_custom"]
params = grid_result.cv_results_["params"]
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))
    
# plot results
scores = np.array(means).reshape(len(max_depth), len(n_estimators))
for i, value in enumerate(max_depth):
    pyplot.plot(n_estimators, scores[i], label='depth: ' + str(value))
pyplot.legend()
pyplot.xlabel('n_estimators')
pyplot.ylabel('JData score')
# pd.DataFrame(grid_result.cv_results_).to_csv('../reports/search_estimators_and_max_depth_all_merged_1.0_rus_0.01.csv', index=False)
pd.DataFrame(grid_result.cv_results_).sort_values('rank_test_custom')<jupyter_output><empty_output><jupyter_text>traning whole dataset -> worse<jupyter_code>param = {'max_depth': 3, 'n_estimators': 350}
clf = XGBClassifier(nthread=-1, **param)

X, y = get_X_y(train)
%time clf.fit(X, y)
clf
# use our best model to evalute result of the whole local data
scoring, scorer = get_jdata_scoring(train)
scores = scoring['custom_index'](clf, X, y, np.arange(X.shape[0]))
print(f'whole local result: {scores}')<jupyter_output>whole local result: 0.04065112543626942<jupyter_text>use model trained by sampling dataset to submit -> not good enough<jupyter_code>param = {'max_depth': 3, 'n_estimators': 350}
clf = XGBClassifier(nthread=-1, **param)
%time clf.fit(res_X, res_y)

scoring, scorer = get_jdata_scoring(train)
scores = scoring['custom_index'](clf, X, y, np.arange(X.shape[0]))
print(f'whole local result: {scores}')
score_df = score_whole_dataset(clf, online_train)
score_df
# trytry place
# rank3
param = {'max_depth': 5, 'n_estimators': 350}
clf = XGBClassifier(nthread=-1, **param)
%time clf.fit(res_X, res_y)

scoring, scorer = get_jdata_scoring(train)
scores = scoring['custom_index'](clf, X, y, np.arange(X.shape[0]))
print(f'whole local result: {scores}')
score_df = score_whole_dataset(clf, online_train)
score_df
## 增加深度 local score 增加但是 online 分數沒有增加，代表 feature 無法提供足夠的資訊讓 model 預測？
## overfitting?
## ensemble to reduce overfitting
## sampling 3 sample datasets to ensemble results
import random

rslt = []
for i in range(3):
    rs = {}
    method = 'rus'
    ratio = 0.01
    random_state = random.randint(0, 10000)
    sampler_setting = {"name": method, "ratio": ratio, "random_state": random_state}
    print(sampler_setting)
    s = sampler(**sampler_setting)
    res_X, res_y, indices = s.fit_sample(X, y)
    
    rs['method'] = method
    rs['ratio'] = ratio
    rs['random_state'] = random_state
    rs['indices'] = indices
    
    param = {'max_depth': 3, 'n_estimators': 350}
    clf = XGBClassifier(nthread=-1, **param)
    %time clf.fit(res_X, res_y)
    
    rs['param'] = param
    rs['model'] = clf

    scoring, scorer = get_jdata_scoring(train)
    scores = scoring['custom_index'](clf, X, y, np.arange(X.shape[0]))
    print(f'whole local result: {scores}')
    
    rs['scoring'] = scoring
    rs['scorer'] = scorer
    rslt.append(rs)
from sklearn.ensemble import VotingClassifier
eclf = VotingClassifier(estimators=[('xgb1', rslt[0]['model']), ('xgb2', rslt[1]['model']), ('xgb3', rslt[2]['model'])], voting='soft')
%time eclf = eclf.fit(res_X, res_y)
score_whole_dataset(eclf, online_train)
# TODO
# remove unseen sku?<jupyter_output><empty_output><jupyter_text>plot importance<jupyter_code>feature_names = train.drop(columns=["user_id", "label"]).columns
feature_mapping = dict([('f{}'.format(i), feature_names[i]) for i in range(len(feature_names))])
from sklearn import preprocessing

def plot_importance(model, feature_mapping, n=30):

    # Get xgBoost importances
    importance_dict = {}
    for import_type in ['weight', 'gain', 'cover']:
        importance_dict['xgBoost-{}'.format(import_type)] = model.get_booster().get_score(importance_type=import_type)

    # MinMax scale all importances
    importance_df = pd.DataFrame(importance_dict).fillna(0)
    importance_df = pd.DataFrame(
        preprocessing.MinMaxScaler().fit_transform(importance_df),
        columns=importance_df.columns,
        index=importance_df.index
    )

    # replace index
    importance_df.index = importance_df.index.map(mapper=feature_mapping)

    # Create mean column
    importance_df['mean'] = importance_df.mean(axis=1)

    # Plot the feature importances
    importance_df.sort_values('mean').head(n).plot(kind='bar', figsize=(20, 10))
plot_importance(clf, feature_mapping)<jupyter_output>/home/makalon/.pyenv/versions/3.7.0/lib/python3.7/site-packages/sklearn/preprocessing/data.py:323: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by MinMaxScaler.
  return self.partial_fit(X, y)
{'fraction_non_alphanumeric': 0.13902872371927746, 'fraction_numerical': 0.022653242522949365, 'mean_word_length': 5.225345622119816, 'pattern_counts': {'":': 18, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '33643', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 5114, 'n_tokens_neox': 4842, 'n_words': 972, 'path': 'notebooks/4. Model Evaluation.ipynb', 'repo_name': 'paristsai/jdata', 'size': '13508'}