<jupyter_start><jupyter_text>Generating Dataset<jupyter_code>X_train, y_train, X_test, y_test = generate_data()<jupyter_output><empty_output><jupyter_text>Applying Decision Tree before PCA<jupyter_code>def DT_before_PCA():
    
    p_grid = {'criterion': ['gini','entropy'],
                   'max_depth': [2,3,4,5,6,7],
                   'min_samples_split':[2,3,4,5,10,20,50],
                   }
    clf = DecisionTreeClassifier()
    clf_random = RandomizedSearchCV(estimator=clf, param_distributions=p_grid,
                             n_iter=100, cv = 3, n_jobs = -1, scoring='recall')
    clf_random.fit(X_train, y_train)
    #     cv = ShuffleSplit(n_splits=10, test_size=0.33, random_state=0)
    scores = cross_val_score(clf_random, X_train, y_train, cv=3)
    val_score = (scores.mean(), scores.std() * 2)

    y_pred = clf_random.predict(X_test)
    conf_mat=confusion_matrix(y_test,y_pred)
    outcome = list(generate_report(conf_mat, y_test, y_pred))
    outcome.insert(0, val_score[0])
#     save_model(outcome, DT_before_PCA)
#     print(clf_random.best_estimator_)
    return outcome, clf_random<jupyter_output><empty_output><jupyter_text>Calculating n_components in PCA for feature reduction<jupyter_code>#Centering data by mean i.e. subtracting mean from each data and doing PCA
# X_train, y_train, X_test, y_test = generate_data()
scaler=StandardScaler().fit(X_train)
X_train=scaler.transform(X_train)
X_test=scaler.transform(X_test)


covar_matrix=PCA()
covar_matrix.fit(X_train)
variance = covar_matrix.explained_variance_ratio_ #calculate variance ratios
var=np.cumsum(np.round(covar_matrix.explained_variance_ratio_, decimals=3)*100)
print(var) #cumulative sum of variance explained with [n] features

plt.ylabel('% Variance Explained')
plt.xlabel('# of Features')
plt.title('PCA Analysis')
plt.ylim(50,100.5)
plt.xlim(0,10)
plt.style.context('seaborn-whitegrid')


plt.plot(var, marker='o')
def apply_PCA(X_train, X_test):
    # we can reduce number of features to 5 to capture 99.8 % variance in PCA
    pca=PCA(n_components=5)
    pca.fit(X_train)

    # Get new feature matrices from pca
    pca_X_train=pca.transform(X_train)
    pca_X_test=pca.transform(X_test)
    return pca_X_train, pca_X_test
pca_X_train, pca_X_test = apply_PCA(X_train, X_test)<jupyter_output><empty_output><jupyter_text>Running Decision Tree Algorithm after PCA<jupyter_code>def DT_after_PCA():
    scoring = ['accuracy', 'recall']
#     X_train, y_train, X_test, y_test = generate_data()
#     pca_X_train, pca_X_test = apply_PCA(X_train, X_test)
    

    p_grid = {'criterion': ['gini','entropy'],
               'max_depth': [2,3,4,5,6,7,8,9],
               'min_samples_split':[2,3,4,5],
               }

    clf = DecisionTreeClassifier()
    clf_gs = RandomizedSearchCV(estimator=clf, param_distributions=p_grid,
                                n_iter=100, cv = 3, n_jobs = -1, scoring='f1')
    clf_gs.fit(X_train, y_train)
    scores = cross_val_score(clf_gs, X_train, y_train, cv=3)
    val_score = (scores.mean(), scores.std() * 2)

    y_pred = clf_gs.predict(X_test)
    conf_mat=confusion_matrix(y_test,y_pred)
    outcome = list(generate_report(conf_mat, y_test, y_pred))
    outcome.insert(0, val_score[0])
#     save_model(outcome, DT_before_PCA)

#     print(clf_gs.best_estimator_)

    return outcome, clf_gs<jupyter_output><empty_output><jupyter_text>Running Logistic Regression after PCA<jupyter_code>def LR_after_PCA():
#     X_train, y_train, X_test, y_test = generate_data()
#     pca_X_train, pca_X_test = apply_PCA(X_train, X_test)

#     scoring = ['accuracy', 'f1']
    p_grid = {'solver' : ['newton-cg','lbfgs', 'liblinear', 'sag', 'saga'],
                   'C' : [.1,.2,.3,.4,.5,.6,.7,.8,.9,1],
                   'max_iter':[100,200,300],
                   
                  }
    
    clf = LogisticRegression()
    clf_random = RandomizedSearchCV(estimator=clf, param_distributions=p_grid,
                                n_iter=100, cv = 3, n_jobs = -1, scoring='f1')
    clf_random.fit(pca_X_train,y_train)

    # Trying ShuffleSplit
#     cv = ShuffleSplit(n_splits=10, test_size=0.33, random_state=0)
    scores = cross_val_score(clf_random, pca_X_train, y_train, cv=5)
#     print("cross_val_score Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))

    y_pred = clf_random.predict(pca_X_test)
    conf_mat=confusion_matrix(y_test,y_pred)
    
    outcome = list(generate_report(conf_mat, y_test, y_pred))
    outcome.insert(0, scores[0])
#     save_model(outcome, DT_before_PCA)

#     print(clf_random.best_estimator_)
    return outcome, clf_random<jupyter_output><empty_output><jupyter_text>Running Linear SVM after PCA<jupyter_code>def SVM_after_PCA():
#     start = time.perf_counter()
#     X_train, y_train, X_test, y_test = generate_data()
    
#     p_grid = {
#                    'C' : [.2, .3, .4, .5, .6, .7, .8, .9, 1],
#                   'kernel': ['linear','poly','rbf','sigmoid'],
#                   'degree': [2,3,4],
#                   'gamma': ['scale','auto'],
#                   'decision_function_shape': ['ovo','ovr'],
#                   }
    p_grid = {
                   'C' : [.2, .3, .4, .5, .6, .7, .8, .9, 1],
                  'kernel': ['linear','poly','rbf','sigmoid'],
                  }
    
    clf = SVC()
    clf_random = RandomizedSearchCV(estimator=clf, param_distributions=p_grid, 
                                    n_iter = 100, cv = 3, random_state=42, 
                                    n_jobs = -1, scoring='f1')
#     clf_gs = GridSearchCV(estimator=clf, param_grid=p_grid, 
#                                     cv = 3, n_jobs = -1, scoring='f1')
    clf_random.fit(pca_X_train,y_train)

    # Trying ShuffleSplit
#     cv = ShuffleSplit(n_splits=10, test_size=0.33, random_state=0)
    scores = cross_val_score(clf_random, pca_X_train, y_train, cv=3)
#     print("cross_val_score Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))

    y_pred = clf_random.predict(pca_X_test)
    conf_mat=confusion_matrix(y_test,y_pred)
    
    outcome = list(generate_report(conf_mat, y_test, y_pred))
    outcome.insert(0, scores[0])
    

    return outcome, clf_random
    
def progressbar(it, prefix="", size=60, file=sys.stdout):
    count = len(it)
    def show(j):
        x = int(size*j/count)
        file.write("%s[%s%s] %i/%i\r" % (prefix, "#"*x, "."*(size-x), j, count))
        file.flush()        
    show(0)
    for i, item in enumerate(it):
        yield item
        show(i+1)
    file.write("\n")
    file.flush()<jupyter_output><empty_output><jupyter_text>Getting the report of different algorithms<jupyter_code>def start_training(max_experiment=5):
    # algos = {'DT before PCA':DT_before_PCA,
    #          'DT with PCA':DT_after_PCA,
    #          'LR with PCA':LR_after_PCA,
    #          'SVM with PCA':SVM_after_PCA}

    algos = {'DT before PCA':DT_before_PCA,
             'DT with PCA':DT_after_PCA,
            }
    
    expected_accuracy = 0.997
    expected_FN = 2
    

    count = max_experiment # Experiment Count for overall analysis

    for c in progressbar(range(count), "Training progress: ", 20):
#         if 'DT with PCA_model.model' or 'DT before PCA_model.model' in glob.glob("*.model"):
#             break
#     for c in range(count):
        X_train, y_train, X_test, y_test = generate_data()        
        pca_X_train, pca_X_test = apply_PCA(X_train, X_test)
        print('\n')
        start = time.perf_counter()

        measurements = []
        for k in algos.keys():
            algo = algos[k]
            outcome, clf = algo()
            measurements.append(outcome)
            if outcome[3] > expected_accuracy and outcome[2] <= expected_FN: # outcome[2] -> False Negative and  outcome[3] -> Prediction Accuracy
                # Save the model for future use 
                filename = k + '_model.model'
                joblib.dump(clf, filename)


        print("\nRan the experiment with dataset no.", c+1)
        print("Report: \n")
        cols = ['Val_Accuracy', 'FP', 'FN', 'Accuracy', 'Recall', 'Precision', 'F1Score']

        df_outcome = pd.DataFrame(measurements, columns=cols)
        df_outcome.index = list(algos.keys()) 
        print(df_outcome)

        finish = time.perf_counter()
        print("Time Taken for execution is ", round(finish-start, 2), 'Seconds\n\n')
    if 'DT with PCA_model.model' or 'DT before PCA_model.model' not in glob.glob("*.model"):
        print('Experimented ', count,'times for attaining expected accuracy. Start training again for another try')
    print('Experiment Completed')
i = 1
while 'DT with PCA_model.model' or 'DT before PCA_model.model' not in glob.glob("*.model"):
    
    start_training()
    time.sleep(0.5)
trained_model = joblib.load("DT with PCA_model.model")
# if 'DT with PCA_model.sav' or 'DT before PCA_model.sav' in glob.glob("*.sav"):
#     print('yes')<jupyter_output><empty_output><jupyter_text>Testing stage for evaluation of the trained model!<jupyter_code>for _ in range(100):
    X_train, y_train, X_test, y_test = generate_data() # Generate random dataset 
    pca_X_train, pca_X_test = apply_PCA(X_train, X_test) # Apply PCA

    y_pred = trained_model.predict(X_test)
    conf_mat=confusion_matrix(y_test,y_pred)

    outcome = list(generate_report(conf_mat, y_test, y_pred))
    print(outcome)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13386496544391283, 'fraction_numerical': 0.019351408825093036, 'mean_word_length': 3.066580198875919, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '149392', 'license': 'MIT', 'n_tokens_mistral': 3586, 'n_tokens_neox': 3418, 'n_words': 632, 'path': 'old source code/hp_optzed_report_ready_01Oct.ipynb', 'repo_name': 'rahulrajpl/Train-Until-Best-Estimater-for-DT-Classifier', 'size': '9405'}