<jupyter_start><jupyter_text>Classification<jupyter_code>import itertools
import matplotlib.gridspec as gridspec
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, ExtraTreesClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split, GridSearchCV, StratifiedKFold
from sklearn.metrics import mean_squared_error, accuracy_score, roc_curve, auc
from sklearn.preprocessing import LabelEncoder
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.neural_network import MLPClassifier
from xgboost.sklearn  import XGBClassifier
from mlxtend.classifier import StackingClassifier
from mlxtend.plotting import plot_decision_regions
y_train = train['outliers']
del train['outliers']
train = train.drop(['target'],axis = 1)
test = test.drop(['target'],axis = 1)
train['target'] = target
test['target'] = predictions_3
clf1 = ExtraTreesClassifier(n_estimators= 500,
                             max_features= 'auto',
                             criterion= 'entropy',
                             min_samples_split= 2,
                             max_depth= 14,
                             min_samples_leaf= 1, 
                             class_weight='balanced_subsample',
                             random_state=132,
                             verbose=0,
                             n_jobs = -1)

cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcomes = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf1.fit(train.iloc[x], y_train.iloc[x]) 
    predictions = clf1.predict(Xtest)
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred1 = clf1.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome)) 
clf2 = ExtraTreesClassifier(n_estimators=500,
                             max_features= 'auto',
                             criterion= 'entropy',
                             min_samples_split= 3,
                             max_depth= 15,
                             min_samples_leaf= 3, 
                             class_weight='balanced_subsample',
                             random_state=1,
                             verbose=0,
                             n_jobs = -1)

cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcome = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):    
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf2.fit(train.iloc[x], y_train.iloc[x]) 
    predictions = clf2.predict(Xtest)
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred2 = clf2.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome))
clf4 = RandomForestClassifier(
    n_estimators=500,
    criterion='gini',
    max_depth= 7,
    min_samples_split=5,
    min_samples_leaf=5,
    min_weight_fraction_leaf=0.0,
    max_features=15,
    max_leaf_nodes=None,
    min_impurity_decrease=0.0,
    min_impurity_split=None,
    bootstrap=True,
    oob_score=False,
    n_jobs=-1,
    random_state=123,
    verbose=0,
    warm_start=False,
    class_weight='balanced')

cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcomes = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf4.fit(train.iloc[x], y_train.iloc[x]) 
    predictions = clf4.predict(Xtest)
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred4 = clf4.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome)) 
clf5 = AdaBoostClassifier(base_estimator=clf1, 
                          n_estimators= 50, 
                          learning_rate=0.1,
                          depth=3,
                          random_state = 0)

cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcomes = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf5.fit(train.iloc[x], y_train.iloc[x]) 
    predictions = clf5.predict(Xtest)
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred5 = clf5.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome)) 
sclf_target = StackingClassifier(classifiers=[clf1, clf4, clf5], 
                          meta_classifier=XGBClassifier(random_state = 1))
sclf_target.fit(train, y_train)
y_pred_target = sclf_target.predict(test)
clf3 = ExtraTreesClassifier(n_estimators=500,
                             max_features= 'auto',
                             criterion= 'entropy',
                             min_samples_split= 3,
                             max_depth= 28,
                             min_samples_leaf= 3, 
                             class_weight='balanced_subsample',
                             random_state=1,
                             verbose=0,
                             n_jobs = -1)

cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcome = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):    
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf3.fit(train.iloc[x], y_train.iloc[x]) 
    predictions = clf3.predict(Xtest)
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred3 = clf3.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome))
clf7 = RandomForestClassifier(
    n_estimators=500,
    criterion='gini',
    max_depth= 60,
    min_samples_split=2,
    min_samples_leaf=2,
    min_weight_fraction_leaf=0.0,
    max_features=15,
    max_leaf_nodes=None,
    min_impurity_decrease=0.0,
    min_impurity_split=None,
    bootstrap=True,
    oob_score=False,
    n_jobs=-1,
    random_state=123,
    verbose=0,
    warm_start=False,
    class_weight='balanced')

cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcomes = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf7.fit(train.iloc[x], y_train.iloc[x]) 
    predictions = clf7.predict(Xtest)
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred7 = clf7.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome)) 
import re
regex = re.compile(r"\[|\]|<", re.IGNORECASE)
train.columns = [regex.sub("_", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in train.columns.values]
test.columns = [regex.sub("_", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in test.columns.values]
test = test[train.columns]

clf6 = XGBClassifier(learning_rate =0.1,
                     n_estimators=300,
                     max_depth = 8,
                     min_child_weight=3,
                     gamma=0,
                     subsample=0.8,
                     colsample_bytree=0.8,
                     objective= 'binary:logistic',
                     nthread=4,
                     scale_pos_weight=1,
                     seed=13)
cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True)
outcome = []
    
for (x, y), i in zip(cv.split(train, y_train), range(5)):
    Xtrain, Xtest = train.values[x], train.values[y]
    ytrain, ytest = y_train.values[x], y_train.values[y]
    clf6.fit(train.iloc[x], y_train.iloc[x])
    predictions = clf6.predict(train.iloc[y])
    accuracy = accuracy_score(ytest, predictions)
    outcomes.append(accuracy)

pred6 = clf6.predict(test)
mean_outcome = np.mean(outcomes)
print("Mean Accuracy: {0}".format(mean_outcome))
pred = np.round((pred1 + pred2 + pred3 + pred4 + pred5 + pred6 + pred7)/7)
sclf_nontarget = StackingClassifier(classifiers=[clf3, clf6, clf7], 
                          meta_classifier=AdaBoostClassifier(random_state = 1))
sclf_nontarget.fit(train, y_train)
y_pred_nontarget = sclf_nontarget.predict(test)
r_pred = np.zeros(123623)
for i in Rxbg1.x:
    r_pred[i-1] = 1
ensemble = np.round((pred + y_pred_target + y_pred_nontarget)/3)
ensemble = np.round((pred1 + pred2)/2 - 0.01)
(pred6 == 1).sum()
sample_submission['outliers'] = pred
sample_submission.loc[sample_submission['outliers'] == 1, 'target'] = -33.218750
sample_submission = sample_submission.drop(['outliers'], axis = 1)
sample_submission.to_csv('submission.csv', index=False)
sample_submission['outliers'] = pred
sample_submission.loc[(sample_submission['outliers'] == 1)]
Rxbg1 = pd.read_csv('prediction.csv')
sample_submission['outliers'] = 0
for i in Rxbg1.x:
    sample_submission['outliers'][i-1] = 1
sample_submission.loc[sample_submission['outliers'] == 1]
# sample_submission.loc[sample_submission['outliers'] == 1, 'target'] = -33.218750
# sample_submission = sample_submission.drop(['outliers'], axis = 1)
# sample_submission.to_csv('submission.csv', index=False)
sub1 = pd.read_csv('82.csv')
sub2 = pd.read_csv('82 (2).csv')
sample_submission = pd.read_csv('sample_submission.csv')
sample_submission['target'] = sub1['target']*0.5 + sub2['target']*0.5
sample_submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12183607874648453, 'fraction_numerical': 0.02450783447167537, 'mean_word_length': 3.0674019607843137, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '481400', 'license': 'MIT', 'n_tokens_mistral': 3610, 'n_tokens_neox': 3438, 'n_words': 598, 'path': 'Elo merchant/Elo_mixer1-Copy1.ipynb', 'repo_name': 'nguyenphuhien13/Kaggle', 'size': '9956'}