<jupyter_start><jupyter_text><jupyter_code>from IPython.display import clear_output
from google.colab import drive
drive.mount('/content/drive')
import sys
import os
import numpy as np
import pandas as pd
import seaborn as sns
import sklearn as sk
from pathlib import PurePath
from pandas import DataFrame
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, Normalizer
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2, f_regression
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.linear_model import LinearRegression
sys.path.append('/content/drive/My Drive/Colab/l2_ML/')
% matplotlib inline
np.random.seed(0)
clear_output()
print(sk.metrics.SCORERS.keys())
folder_path = '/content/drive/My Drive/Colab/l2_ML/'
csv_filename = os.path.join(folder_path ,'unspaced_bp.csv.gz')
df = pd.read_csv(csv_filename, compression='gzip')
print(df.columns)
print(df.head())
le_sys = LabelEncoder()
X_sys = df.drop(['pid', 'cid', 'sys', 'dia', 'sys_labels', 'dia_labels'], axis=1)
y_sys = le_sys.fit_transform(df['sys_labels'])
X_dia = df.drop(['pid', 'cid', 'sys', 'dia', 'sys_labels', 'dia_labels'], axis=1)
y_dia = df['dia_labels']
clf = RandomForestClassifier(random_state=0)
params = {'n_estimators': range(10, 50, 10), 'max_depth': range(1, 12, 2), \
          'min_samples_leaf': range(1, 7, 1), 'min_samples_split': range(2, 9, 1)}
search = GridSearchCV(clf, param_grid=params, scoring='accuracy', n_jobs=-1, cv=5)
search.fit(X_sys, y_sys)
# print(search.score())
cross_val_scores = cross_val_score(search.best_estimator_, X_sys, y_sys, cv=5, scoring='accuracy')
print(cross_val_scores)
print(cross_val_scores.mean())
cross_val_scores = cross_val_score(search.best_estimator_, X_sys, y_sys, cv=5, scoring='f1_samples')
print(cross_val_scores)
print(cross_val_scores.mean())<jupyter_output>[0.80413625 0.80389769 0.80389769 0.80487805 0.80487805]
0.8043375444201022
{'fraction_non_alphanumeric': 0.13022113022113022, 'fraction_numerical': 0.044226044226044224, 'mean_word_length': 7.627118644067797, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '276253', 'license': 'Apache-2.0', 'n_tokens_mistral': 797, 'n_tokens_neox': 729, 'n_words': 156, 'path': 'BP_classifier.ipynb', 'repo_name': 'bvm84/open_colab_ml', 'size': '2035'}