<jupyter_start><jupyter_text>Evaluation:<jupyter_code>performance.columns
# label encoding the data 
from sklearn.preprocessing import LabelEncoder 
le = LabelEncoder() 
bottom_list_df[0]= le.fit_transform(bottom_list_df[0]) 
bottom_list_df[1]= le.fit_transform(bottom_list_df[1]) 


bottom_list_df[2]= le.fit_transform(bottom_list_df[2]) 
bottom_list_df[3]= le.fit_transform(bottom_list_df[3]) 

bottom_list_df[4]= le.fit_transform(bottom_list_df[4]) 
bottom_list_df[5]= le.fit_transform(bottom_list_df[5]) 

bottom_list_df[6]= le.fit_transform(bottom_list_df[6]) 
bottom_list_df[7]= le.fit_transform(bottom_list_df[7]) 

bottom_list_df[8]= le.fit_transform(bottom_list_df[8]) 
bottom_list_df[9]= le.fit_transform(bottom_list_df[9]) 

bottom_list_df[10]= le.fit_transform(bottom_list_df[10]) 
bottom_list_df[11]= le.fit_transform(bottom_list_df[11]) 

bottom_list_df[12]= le.fit_transform(bottom_list_df[12]) 
bottom_list_df[13]= le.fit_transform(bottom_list_df[13]) 

bottom_list_df[14]= le.fit_transform(bottom_list_df[14]) 
bottom_list_df[15]= le.fit_transform(bottom_list_df[15]) 

bottom_list_df[16]= le.fit_transform(bottom_list_df[16]) 
bottom_list_df[17]= le.fit_transform(bottom_list_df[17]) 

bottom_list_df[18]= le.fit_transform(bottom_list_df[18]) 
bottom_list_df[19]= le.fit_transform(bottom_list_df[19]) 
bottom_list_df.head()
top_list_df[0]= le.fit_transform(top_list_df[0]) 
top_list_df[1]= le.fit_transform(top_list_df[1]) 

top_list_df[2]= le.fit_transform(top_list_df[2]) 
top_list_df[3]= le.fit_transform(top_list_df[3]) 

top_list_df[4]= le.fit_transform(top_list_df[4]) 
top_list_df[5]= le.fit_transform(top_list_df[5]) 

top_list_df[6]= le.fit_transform(top_list_df[6]) 
top_list_df[7]= le.fit_transform(top_list_df[7]) 

top_list_df[8]= le.fit_transform(top_list_df[8]) 
top_list_df[9]= le.fit_transform(top_list_df[9]) 

top_list_df[10]= le.fit_transform(top_list_df[10]) 
top_list_df[11]= le.fit_transform(top_list_df[11]) 

top_list_df[12]= le.fit_transform(top_list_df[12]) 
top_list_df[13]= le.fit_transform(top_list_df[13]) 

top_list_df[14]= le.fit_transform(top_list_df[14]) 
top_list_df[15]= le.fit_transform(top_list_df[15]) 

top_list_df[16]= le.fit_transform(top_list_df[16]) 
top_list_df[17]= le.fit_transform(top_list_df[17]) 

top_list_df[18]= le.fit_transform(top_list_df[18]) 
top_list_df[19]= le.fit_transform(top_list_df[19]) 
top_list_df.tail()
top_list_df.shape
bottom_list_df.shape
performance.shape
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.pipeline import Pipeline
pipeline = Pipeline([
    #('normalizer', StandardScaler()), #Step1 - normalize data
    ('clf', LogisticRegression()) #step2 - classifier
])
pipeline.steps
x=0
y=520
data=performance[x:y]
data.shape<jupyter_output><empty_output><jupyter_text>Method-1  top_list_Data<jupyter_code>from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(top_list_df,data,test_size=0.20,random_state=2)
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
from sklearn.model_selection import cross_validate
scores = cross_validate(pipeline, X_train, y_train)
scores
scores['test_score'].mean()

from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

clfs = []
clfs.append(LogisticRegression())
clfs.append(SVC())
clfs.append(SVC())
clfs.append(KNeighborsClassifier(n_neighbors=3))
clfs.append(DecisionTreeClassifier())
clfs.append(RandomForestClassifier())
clfs.append(GradientBoostingClassifier())

for classifier in clfs:
    pipeline.set_params(clf = classifier)
    scores = cross_validate(pipeline, X_train, y_train)
    print('---------------------------------')
    print(str(classifier))
    print('-----------------------------------')
    for key, values in scores.items():
            print(key,' mean ', values.mean())
            print(key,' std ', values.std())<jupyter_output>/home/hemanth/.local/lib/python3.8/site-packages/sklearn/utils/validation.py:72: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  return f(**kwargs)
/home/hemanth/.local/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
/home/hemanth/.local/lib/python3.8/site-packages/sklearn/utils/validation.py:72: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example [...]<jupyter_text>Bottom_list_data<jupyter_code>pipeline1 = Pipeline([
    
    ('clf', LogisticRegression()) #step2 - classifier
])
pipeline1.steps
X_train,X_test,y_train,y_test=train_test_split(bottom_list_df,data,test_size=0.30,random_state=2)
scores = cross_validate(pipeline1, X_train, y_train)
scores
scores['test_score'].mean()
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

clfs = []
clfs.append(LogisticRegression())
clfs.append(SVC())
clfs.append(SVC())
clfs.append(KNeighborsClassifier(n_neighbors=3))
clfs.append(DecisionTreeClassifier())
clfs.append(RandomForestClassifier())
clfs.append(GradientBoostingClassifier())

for classifier in clfs:
    pipeline1.set_params(clf = classifier)
    scores = cross_validate(pipeline1, X_train, y_train)
    print('---------------------------------')
    print(str(classifier))
    print('-----------------------------------')
    for key, values in scores.items():
            print(key,' mean ', values.mean())
            print(key,' std ', values.std())<jupyter_output>/home/hemanth/.local/lib/python3.8/site-packages/sklearn/utils/validation.py:72: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  return f(**kwargs)
/home/hemanth/.local/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
/home/hemanth/.local/lib/python3.8/site-packages/sklearn/utils/validation.py:72: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example [...]<jupyter_text>Method-2  top_list-df  Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
X_train,X_test,y_train,y_test=train_test_split(top_list_df,data,test_size=0.3,random_state=2)
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
logreg=LogisticRegression()
model_logreg=logreg.fit(X_train,y_train)
logreg_predict= model_logreg.predict(X_test)
from sklearn.metrics import accuracy_score,confusion_matrix,classification_report
logistic=accuracy_score(logreg_predict,y_test)
logistic
print(classification_report(logreg_predict,y_test))
cm = confusion_matrix(y_test, logreg_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
X_train,X_test,y_train,y_test=train_test_split(top_list_df,data,test_size=0.25,random_state=2)
knn=KNeighborsClassifier(n_neighbors=3)
model_knn= knn.fit(X_train,y_train)
knn_predict=model_knn.predict(X_test)
knn=accuracy_score(knn_predict,y_test)
knn
print(classification_report(y_test,knn_predict))
cm = confusion_matrix(y_test, knn_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>SVM<jupyter_code>from sklearn.svm import SVC
svc=SVC()
model_svm=svc.fit(X_train,y_train)
svm_predict=model_svm.predict(X_test)
svm=accuracy_score(svm_predict,y_test)
svm
print(classification_report(svm_predict,y_test))
cm = confusion_matrix(y_test, svm_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>DT<jupyter_code>from sklearn.tree import DecisionTreeClassifier
dtree=DecisionTreeClassifier()
model_dt=dtree.fit(X_train,y_train)
dtree_predict=model_dt.predict(X_test)
dt=accuracy_score(dtree_predict,y_test)
dt
print(classification_report(dtree_predict,y_test))
cm = confusion_matrix(y_test, dtree_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>RF<jupyter_code>from sklearn.ensemble import RandomForestClassifier
rfc=RandomForestClassifier()
model_rf=rfc.fit(X_train,y_train)
rfc_predict=model_rf.predict(X_test)
rf=accuracy_score(rfc_predict,y_test)
rf
print(classification_report(rfc_predict,y_test))
cm = confusion_matrix(y_test, rfc_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>AdaBoosting<jupyter_code>from sklearn.ensemble import AdaBoostClassifier
adc=AdaBoostClassifier(n_estimators=5,learning_rate=1)
model_adaboosting=adc.fit(X_train,y_train)
adc_predict=model_adaboosting.predict(X_test)
adaboostin=accuracy_score(adc_predict,y_test)
adaboostin
print(classification_report(adc_predict,y_test))
cm = confusion_matrix(y_test, adc_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>XGBoosting<jupyter_code>from xgboost import XGBClassifier
xgb=XGBClassifier()
model_xgb=xgb.fit(X_train,y_train)
xgb_predict=model_xgb.predict(X_test)
xgb=accuracy_score(xgb_predict,y_test)
xgb
print(classification_report(xgb_predict,y_test))
cm = confusion_matrix(y_test, xgb_predict)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)
pd.DataFrame({"Model Names":['Logistic Reg','SVM','KNN','DT','RF','ADABoostin','XGB'],
            "Accuracy socre":[logistic,svm,knn,dt,rf,adaboostin,xgb]})<jupyter_output><empty_output><jupyter_text>Bottom_list_df  Log_Reg<jupyter_code>from sklearn.linear_model import LogisticRegression
X_train,X_test,y_train,y_test=train_test_split(bottom_list_df,data,test_size=0.3,random_state=2)
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
logreg_b=LogisticRegression()
model_logreg_b=logreg_b.fit(X_train,y_train)
logreg_predict_b= model_logreg_b.predict(X_test)
logistic_b=accuracy_score(logreg_predict_b,y_test)
logistic_b
print(classification_report(logreg_predict_b,y_test))
cm = confusion_matrix(y_test, logreg_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)
knn=KNeighborsClassifier(n_neighbors=3)
model_knn_b= knn.fit(X_train,y_train)
knn_predict_b=model_knn_b.predict(X_test)

knn_b=accuracy_score(knn_predict_b,y_test)
knn_b
print(classification_report(y_test,knn_predict_b))
cm = confusion_matrix(y_test, knn_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>SVM<jupyter_code>svc=SVC()
model_svm_b=svc.fit(X_train,y_train)
svm_predict_b=model_svm_b.predict(X_test)
svm_b=accuracy_score(svm_predict_b,y_test)
svm_b
print(classification_report(y_test,svm_predict_b))
cm = confusion_matrix(y_test, svm_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>DT<jupyter_code>dtree=DecisionTreeClassifier()
model_dt_b=dtree.fit(X_train,y_train)
dtree_predict_b=model_dt_b.predict(X_test)
dt_b=accuracy_score(dtree_predict_b,y_test)
dt_b
print(classification_report(dtree_predict_b,y_test))
cm = confusion_matrix(y_test, dtree_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>RF<jupyter_code>rfc=RandomForestClassifier()
model_rf_b=rfc.fit(X_train,y_train)
rfc_predict_b=model_rf_b.predict(X_test)
rf_b=accuracy_score(rfc_predict_b,y_test)
rf_b
print(classification_report(rfc_predict_b,y_test))
cm = confusion_matrix(y_test, rfc_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>ADBoosting<jupyter_code>adc=AdaBoostClassifier(n_estimators=5,learning_rate=1)
model_adaboosting_b=adc.fit(X_train,y_train)
adc_predict_b=model_adaboosting_b.predict(X_test)
adb_b=accuracy_score(adc_predict_b,y_test)
adb_b

print(classification_report(adc_predict_b,y_test))
cm = confusion_matrix(y_test, adc_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)<jupyter_output><empty_output><jupyter_text>XGBOOSting<jupyter_code>xgb=XGBClassifier()
model_xgb_b=xgb.fit(X_train,y_train)
xgb_predict_b=model_xgb_b.predict(X_test)
xgb_b=accuracy_score(xgb_predict_b,y_test)
xgb_b
print(classification_report(xgb_predict_b,y_test))
cm = confusion_matrix(y_test, xgb_predict_b)
plt.figure(figsize=(5,5))
p=sns.heatmap(cm, annot=True)
pd.DataFrame({"Model Names_Bottom_list_df":['Logistic Reg','SVM','KNN','DT','RF','ADABoostin','XGB'],
            "Accuracy socre":[logistic_b,svm_b,knn_b,dt_b,rf_b,adb_b,xgb_b]})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12869590058563063, 'fraction_numerical': 0.01571204113698043, 'mean_word_length': 8.896113074204948, 'pattern_counts': {'":': 4, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '383904', 'license': 'MIT', 'n_tokens_mistral': 5628, 'n_tokens_neox': 5423, 'n_words': 684, 'path': 'Pipeline/ML_Task_Classification_Pipeline.ipynb', 'repo_name': 'KarinkiManikanta/Linear-Regression', 'size': '14002'}