<jupyter_start><jupyter_text>Data Preprocess<jupyter_code>def getlistnum(li):
    li = list(li)
    set1 = set(li)
    dict1 = {}
    for item in set1:
        dict1.update({item:li.count(item)})
    return dict1

n_data = data

print(getlistnum(n_data['label']))

n_data = n_data.drop(n_data[n_data.isnull().values==True].index)

print(n_data.isnull().any())
print(np.isnan(np.array(n_data).any()))
print(np.isinf(np.array(n_data).all()))<jupyter_output>{0: 449061, 1: 24817}
trade_date       False
stock_code       False
open             False
high             False
low              False
close            False
pre_close        False
change           False
stock_chg        False
vol              False
amount           False
turnover_rate    False
volume_ratio     False
index_chg        False
gap              False
label            False
dtype: bool
False
False<jupyter_text>To solve the problem of disproportion of ab/non-ab1. select same stock  (fail)        {0: 137413, 1: 24388}        mean:0.2    2. select same stock and abnormal times > 20    {0: 22818, 1: 16197} mean:0.4            Logit 0.43496        SGD 0.54876        Tree 0.87543        3. select same stock and abnormal times > 15    {0: 31840, 1: 17904} mean:0.35    \+ random drop label=0 (index%3==0)        {0: 21200, 1: 17904} mean:0.49            Logit 0.45685        SGD 0.54865        Tree 0.86281    times > 10        \+ random drop label=0 (sqrt(index) % 2 ==0)        {0: 28781, 1: 20299} mean:0.41        Logit 0.40841        SGD 0.40841        Tree 0.86705            times > 5        \+ random drop label=0 index % 4 != 0)        {0: 20634, 1: 22598}  mean:0.521787            Logit acc: 0.51810; pre: 0.51810; recall: 0.51810; F-1: 0.35363; Kappa: 0.00000;        Sgd acc: 0.55129; pre: 0.55129; recall: 0.55129; F-1: 0.47520; Kappa: 0.07730;        Tree acc: 0.84781; pre: 0.84781; recall: 0.84781; F-1: 0.84773; Kappa: 0.69499;            \+ random drop label=0 index % 3 != 0)    {0: 27544, 1: 22598}  mean:0.449979        Logit acc: 0.45348; pre: 0.45348; recall: 0.45348; F-1: 0.28297; Kappa: 0.00000;        SGD acc: 0.45269; pre: 0.45269; recall: 0.45269; F-1: 0.28263; Kappa: -0.00160;        Tree acc: 0.85093; pre: 0.85093; recall: 0.85093; F-1: 0.85091; Kappa: 0.69919;<jupyter_code>ab_code_list = []
ab_code_times = {}
for index, row in n_data.iterrows():
    if row['label']==1:
        if row['stock_code'] in ab_code_times.keys():
            ab_code_times[row['stock_code']] = ab_code_times[row['stock_code']]+1
        else:
            ab_code_times.update({row['stock_code']:1})
        
        if row['stock_code'] in ab_code_list:
            continue
        else:
            ab_code_list.append(row['stock_code'])
print(len(ab_code_list))
input_data = []
GROUP_NUM = 5
group_num = 0
for index, row in n_data.iterrows():
    if row['stock_code'] in ab_code_list:
        if ab_code_times[row['stock_code']] > 5:
            row['group'] = group_num
            if group_num == GROUP_NUM:
                group_num = 0
            group_num = group_num + 1
            input_data.append(row)

i_data = pd.DataFrame(input_data)

print(getlistnum(i_data['label']))
print(len(i_data))
# random drop
drop_list = []
for index, row in i_data.iterrows():
    if row['label'] == 0:
        # toe = int(math.sqrt(index)) % 2
        toe = index % 3
        if toe != 0:
            drop_list.append(index)
new_data = i_data.drop(index = drop_list)
print(getlistnum(new_data['label']))
print(len(new_data))
new_data.to_csv("ab_data_ed.csv")
def getlistnum(li):
    li = list(li)
    set1 = set(li)
    dict1 = {}
    for item in set1:
        dict1.update({item:li.count(item)})
    return dict1

new_data = pd.read_csv("ab_data_ed.csv", index_col=0)
print(getlistnum(new_data['label']))
print(len(new_data))
n_rows = len(new_data)
train_len = int(math.floor(0.80*n_rows))
validation_len = int(n_rows - train_len)

data_input = new_data.reindex(np.random.permutation(new_data.index))

training_input = data_input.head(train_len)
validation_input = data_input.tail(validation_len)
training_targets = training_input[["label"]]
training_examples = training_input.drop(["label","trade_date", "stock_code", "group"], axis=1)

validation_targets = validation_input[["label"]]
validation_examples = validation_input.drop(["label","trade_date", "stock_code", "group"], axis=1)

from IPython import display

print("Training examples summary:")
display.display(training_examples.describe())
print("Validation examples summary:")
display.display(validation_examples.describe())

print("Training targets summary:")
display.display(training_targets.describe())
print("Validation targets summary:")
display.display(validation_targets.describe())<jupyter_output>Training examples summary:<jupyter_text>PCA?Weather use? NOT USE<jupyter_code>from sklearn.decomposition import PCA
pca = PCA(n_components=3)
training_examples = pca.fit_transform(training_examples)
print(pca.explained_variance_ratio_)<jupyter_output>[8.13768586e-01 1.86231410e-01 3.73600066e-09]<jupyter_text>GBDT|FUNCTION|Acc|Pre|Recall|F-1|Kappa||--------|---|---|------|---|-----||USE n=5 | 0.87097 (+/- 0.00392)| 0.86439 | 0.86439 | 0.86439 | 0.86405 | 0.72483 ||USE n=3 | 0.72726 (+/- 0.00212)| 0.70625 | 0.70625 | 0.70625 | 0.70340 | 0.39921 ||NO USE | 0.88714 (+/- 0.00200)| 0.88214 | 0.88214 | 0.88214 | 0.88208 | 0.76160 |  Machine Learningvalidate: GroupShuffleSplitAns|FUNCTION|Acc|Pre|Recall|F-1|Kappa||--------|---|---|------|---|-----||Logit | 0.52327 (+/- 0.00104) | 0.51810 | 0.51810 | 0.51810 | 0.35363 | 0.00000;|SGD | 0.51661 (+/- 0.06175)| 0.55129 | 0.55129 | 0.55129 | 0.47520 | 0.07730;|Decision Tree | 0.84578 (+/- 0.00586)| 0.84781 | 0.84781 | 0.84781 | 0.84773 | 0.69499;|GBDT | 0.88741 (+/- 0.00412)| 0.88632 | 0.88632 | 0.88632 | 0.88635 | 0.77249 |<jupyter_code>from sklearn.model_selection import train_test_split, cross_val_score, GroupShuffleSplit, ShuffleSplit
from sklearn.linear_model import LogisticRegression, LogisticRegressionCV, SGDClassifier
from sklearn import svm, tree
import warnings
warnings.filterwarnings("ignore")
gss = GroupShuffleSplit(n_splits=GROUP_NUM, test_size=0.3, random_state=0)
ss = ShuffleSplit(n_splits=GROUP_NUM, test_size=0.2, random_state=0)
groups = training_input['group'].values

logi_clf = LogisticRegression(solver="lbfgs", C=0.05, penalty="l2").fit(training_examples, training_targets.values)
sgd_clf = SGDClassifier(loss="log", max_iter=10000, alpha=0.01,
                        learning_rate="optimal").fit(training_examples, training_targets.values)
tree_clf = tree.DecisionTreeClassifier().fit(training_examples, training_targets.values)

logit_scores = cross_val_score(logi_clf, training_examples,
                               training_targets.values,
                               cv=gss,groups=groups)

sgd_scores = cross_val_score(sgd_clf, training_examples,
                             training_targets.values,
                             cv=gss,groups=groups)

tree_scores = cross_val_score(tree_clf, training_examples,
                             training_targets.values,
                             cv=gss,groups=groups)

print("Logit Accuracy: %0.5f (+/- %0.5f)" % (logit_scores.mean(), logit_scores.std() * 2 ))
print("SGD Accuracy: %0.5f (+/- %0.5f)" % (sgd_scores.mean(), sgd_scores.std() * 2))
print("Decision Tree Accuracy: %0.5f (+/- %0.5f)" % (tree_scores.mean(), tree_scores.std() * 2))
from sklearn import metrics

logi_validation_ans = logi_clf.predict(validation_examples)
sgd_validation_ans = sgd_clf.predict(validation_examples)
tree_validation_ans = tree_clf.predict(validation_examples)

print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, logi_validation_ans),
       metrics.precision_score(validation_targets, logi_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, logi_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, logi_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, logi_validation_ans)
      ))

print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, sgd_validation_ans,),
       metrics.precision_score(validation_targets, sgd_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, sgd_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, sgd_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, sgd_validation_ans)
      ))

print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, tree_validation_ans),
       metrics.precision_score(validation_targets, tree_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, tree_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, tree_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, tree_validation_ans)
      ))
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import metrics
from sklearn.model_selection import train_test_split, cross_val_score, GroupShuffleSplit, ShuffleSplit
import warnings
warnings.filterwarnings("ignore")
GROUP_NUM = 5
gss = GroupShuffleSplit(n_splits=GROUP_NUM, test_size=0.3, random_state=0)
ss = ShuffleSplit(n_splits=GROUP_NUM, test_size=0.3, random_state=0)
groups = training_input['group'].values

# GBDT_clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0,
#                                  max_depth=1, random_state=0).fit(training_examples, training_targets)

# GBDT_scores = cross_val_score(GBDT_clf, training_examples,
#                              training_targets.values,
#                              cv=gss,groups=groups)
print("GBDT Accuracy: %0.5f (+/- %0.5f)" % (GBDT_scores.mean(), GBDT_scores.std() * 2 ))

GBDT_validation_ans = GBDT_clf.predict(validation_examples)
print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, GBDT_validation_ans),
       metrics.precision_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, GBDT_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, GBDT_validation_ans)
      ))<jupyter_output>GBDT Accuracy: 0.88714 (+/- 0.00200)
| 0.88214 | 0.88214 | 0.88214 | 0.88208 | 0.76160 |<jupyter_text>USE pipelines to determine the para of GBDT<jupyter_code>from sklearn.model_selection import GridSearchCV

parameters = {'n_estimators':range(1,50),
              'learning_rate':[0.01,0.03,0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
              }
GBDT_clf = GradientBoostingClassifier(max_depth = 5)
clf = GridSearchCV(GBDT_clf, parameters, cv = ss)
clf.fit(training_examples, training_targets)
results = pd.DataFrame(clf.cv_results_)
results
lr_001 = [[],[]]
lr_003 = [[],[]]
lr_005 = [[],[]]
lr_01 = [[],[]]
lr_02 = [[],[]]
lr_03 = [[],[]]
lr_04 = [[],[]]
lr_05 = [[],[]]
lr_06 = [[],[]]
lr_07 = [[],[]]
lr_08 = [[],[]]
lr_09 = [[],[]]
lr_1 = [[],[]]
for index, row in results.iterrows():
    if row['param_learning_rate'] == 0.01:
        lr_001[0].append(row['param_n_estimators'])
        lr_001[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.03:
        lr_003[0].append(row['param_n_estimators'])
        lr_003[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.05:
        lr_005[0].append(row['param_n_estimators'])
        lr_005[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.1:
        lr_01[0].append(row['param_n_estimators'])
        lr_01[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.2:
        lr_02[0].append(row['param_n_estimators'])
        lr_02[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.3:
        lr_03[0].append(row['param_n_estimators'])
        lr_03[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.4:
        lr_04[0].append(row['param_n_estimators'])
        lr_04[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.5:
        lr_05[0].append(row['param_n_estimators'])
        lr_05[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.6:
        lr_06[0].append(row['param_n_estimators'])
        lr_06[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.7:
        lr_07[0].append(row['param_n_estimators'])
        lr_07[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.8:
        lr_08[0].append(row['param_n_estimators'])
        lr_08[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 0.9:
        lr_09[0].append(row['param_n_estimators'])
        lr_09[1].append(row['mean_test_score'])
    if row['param_learning_rate'] == 1:
        lr_1[0].append(row['param_n_estimators'])
        lr_1[1].append(row['mean_test_score'])
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
fig = plt.figure()

# 设置三维图形模式
ax = fig.gca(projection='3d')

# 绘制图形
# ax.plot(results['param_learning_rate'].values, 
#         results['param_n_estimators'].values, 
#         results['mean_test_score'].values, label='max_depth = 5')
ax.scatter(results['param_learning_rate'].values, 
        results['param_n_estimators'].values, 
        results['mean_test_score'].values, label='max_depth = 5')

# 显示图例
ax.legend()

# 显示图形
plt.show()

plt.figure(figsize=(16, 8))

plt.plot(lr_001[0], lr_001[1], label="LR=0.01")
plt.plot(lr_003[0], lr_003[1], label="LR=0.03")
plt.plot(lr_005[0], lr_005[1], label="LR=0.05")
plt.plot(lr_01[0], lr_01[1], label="LR=0.1")
plt.plot(lr_02[0], lr_02[1], label="LR=0.2")
plt.plot(lr_03[0], lr_03[1], label="LR=0.3")
plt.plot(lr_04[0], lr_04[1], label="LR=0.4")
plt.plot(lr_05[0], lr_05[1], label="LR=0.5")
plt.plot(lr_06[0], lr_06[1], label="LR=0.6")
plt.plot(lr_07[0], lr_07[1], label="LR=0.7")
plt.plot(lr_08[0], lr_08[1], label="LR=0.8")
plt.plot(lr_09[0], lr_09[1], label="LR=0.9")
plt.plot(lr_1[0], lr_1[1], label="LR=1")

plt.xlim((0, 20))
plt.ylim((0.8,0.95))

plt.xlabel('n_estimators')
plt.ylabel('scores')

plt.legend(loc="upper left")
plt.show()



plt.figure(figsize=(16, 8))

plt.scatter(lr_001[0], lr_001[1], label="LR=0.01")
plt.scatter(lr_003[0], lr_003[1], label="LR=0.03")
plt.scatter(lr_005[0], lr_005[1], label="LR=0.05")
plt.scatter(lr_01[0], lr_01[1], label="LR=0.1")
plt.scatter(lr_02[0], lr_02[1], label="LR=0.2")
plt.scatter(lr_03[0], lr_03[1], label="LR=0.3")
plt.scatter(lr_04[0], lr_04[1], label="LR=0.4")
plt.scatter(lr_05[0], lr_05[1], label="LR=0.5")
plt.scatter(lr_06[0], lr_06[1], label="LR=0.6")
plt.scatter(lr_07[0], lr_07[1], label="LR=0.7")
plt.scatter(lr_08[0], lr_08[1], label="LR=0.8")
plt.scatter(lr_09[0], lr_09[1], label="LR=0.9")
plt.scatter(lr_1[0], lr_1[1], label="LR=1")

plt.xlim((0, 20))
plt.ylim((0.8,0.95))

plt.xlabel('n_estimators')
plt.ylabel('scores')

plt.legend(loc="upper left")
plt.show()
plt.figure(figsize=(16, 8))


plt.plot(lr_03[0], lr_03[1], label="LR=0.3")
plt.plot(lr_04[0], lr_04[1], label="LR=0.4")
plt.plot(lr_05[0], lr_05[1], label="LR=0.5")
plt.plot(lr_06[0], lr_06[1], label="LR=0.6")
plt.plot(lr_07[0], lr_07[1], label="LR=0.7")
plt.plot(lr_08[0], lr_08[1], label="LR=0.8")
plt.plot(lr_09[0], lr_09[1], label="LR=0.9")
plt.plot(lr_1[0], lr_1[1], label="LR=1")

plt.xlim((1, 10))
plt.ylim((0.88,0.92))

plt.xlabel('n_estimators')
plt.ylabel('scores')

plt.legend(loc="upper left")
plt.show()
print(clf.best_params_)
print(clf.refit_time_)
GBDT_clf = GradientBoostingClassifier(n_estimators=15, learning_rate=0.3,
                                 max_depth=5, random_state=0).fit(training_examples, training_targets)
GBDT_scores = cross_val_score(GBDT_clf, training_examples,
                             training_targets.values,
                             cv=gss,groups=groups)
print("GBDT Accuracy: %0.5f (+/- %0.5f)" % (GBDT_scores.mean(), GBDT_scores.std() * 2 ))
GBDT_validation_ans = GBDT_clf.predict(validation_examples)
print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, GBDT_validation_ans),
       metrics.precision_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, GBDT_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, GBDT_validation_ans)
      ))
GBDT_clf = GradientBoostingClassifier(n_estimators=7, learning_rate=0.5,
                                 max_depth=5, random_state=0).fit(training_examples, training_targets)
GBDT_scores = cross_val_score(GBDT_clf, training_examples,
                             training_targets.values,
                             cv=gss,groups=groups)
print("GBDT Accuracy: %0.5f (+/- %0.5f)" % (GBDT_scores.mean(), GBDT_scores.std() * 2 ))
GBDT_validation_ans = GBDT_clf.predict(validation_examples)
print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, GBDT_validation_ans),
       metrics.precision_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, GBDT_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, GBDT_validation_ans)
      ))
GBDT_clf = GradientBoostingClassifier(n_estimators=48, learning_rate=0.1,
                                 max_depth=5, random_state=0).fit(training_examples, training_targets)
GBDT_scores = cross_val_score(GBDT_clf, training_examples,
                             training_targets.values,
                             cv=gss,groups=groups)
print("GBDT Accuracy: %0.5f (+/- %0.5f)" % (GBDT_scores.mean(), GBDT_scores.std() * 2 ))
GBDT_validation_ans = GBDT_clf.predict(validation_examples)
print("| %0.5f | %0.5f | %0.5f | %0.5f | %0.5f |" % 
      (metrics.accuracy_score(validation_targets, GBDT_validation_ans),
       metrics.precision_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.recall_score(validation_targets, GBDT_validation_ans, average='micro'),
       metrics.f1_score(validation_targets, GBDT_validation_ans, average='weighted'),
       metrics.cohen_kappa_score(validation_targets, GBDT_validation_ans)
      ))
import joblib
joblib.dump(GBDT_clf,"AB_Detector_GBDT.model")
clf = joblib.load("AB_Detector_GBDT.model")
clf.predict(np.array([4.10,4.23,4.08,4.23,4.11,0.12,2.9197,3441213.11,1438667.971,0.8322,1.78,0.4727,2.4470]).reshape(1, -1))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16072280218324414, 'fraction_numerical': 0.07959302633670712, 'mean_word_length': 3.966315789473684, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '96457', 'license': 'MIT', 'n_tokens_mistral': 8188, 'n_tokens_neox': 7160, 'n_words': 1079, 'path': 'ab_detector/AnormalyDetector.ipynb', 'repo_name': 'ICEJM1020/DarkPatterns-StockInforPage', 'size': '18871'}