<jupyter_start><jupyter_text>Set upInstall required packages and load relevant data.These functions derived from Roger's prepare data notebook<jupyter_code>import pandas as pd
import numpy as np
from pathlib import Path
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RandomizedSearchCV
from sklearn.decomposition import PCA
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import confusion_matrix
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report, confusion_matrix
from scipy import stats
from joblib import dump
from src.data import make_dataset
from pandas_profiling import ProfileReport
import matplotlib.pyplot as plt
from  matplotlib.ticker import FuncFormatter
import seaborn as sns
project_dir = Path.cwd().parent
report_dir = project_dir / 'reports'
data_dir = project_dir / 'data'
raw_data_dir = data_dir / 'raw'
interim_data_dir = data_dir / 'interim'
processed_data_dir = data_dir / 'processed'
models_dir = project_dir / 'models'<jupyter_output><empty_output><jupyter_text>Download and Prepare Data<jupyter_code>competition = 'uts-advdsi-nba-career-prediction'
make_dataset.download_data(competition=competition,
                           path=raw_data_dir,
                           unzip=True)
df_train = pd.read_csv(raw_data_dir / 'train.csv')
df_train.describe()
X_test = pd.read_csv(raw_data_dir / 'test.csv')
X_test
df_train.drop(columns=['Id_old', 'Id'], inplace=True)
X_test.drop(columns=['Id_old'], inplace=True)
test_id = X_test.pop('Id')<jupyter_output><empty_output><jupyter_text>Exploration<jupyter_code>profile_report = ProfileReport(df_train,
                               title='Raw data report',
                               explorative=True)
profile_report.to_file(report_dir / 'profile_report.html')<jupyter_output><empty_output><jupyter_text>Exploration NotesTODO: summarise  ModellingTODO:* Linear model - exploratory first step* Consider PCA regression - Roger's PCA suggests high colinearity among predictors* Use random forest as a good out of the box tree method to handle colinearity<jupyter_code>target = 'TARGET_5Yrs'
X, y = make_dataset.separate_target(df_train, target=target)
X_train, X_val, y_train, y_val = train_test_split(X, y,
                                                  test_size=0.2,
                                                  random_state=42)<jupyter_output><empty_output><jupyter_text>Random ForestThis is a basic Random Forest with no parameter tuning<jupyter_code>from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(max_depth=3, random_state=0, oob_score = True,class_weight = 'balanced')

clf.fit(X_train, y_train)
y_pred = clf.predict(X_train)
print(classification_report(y_train,y_pred))
confusion_matrix(y_train,y_pred)
y_pred = clf.predict(X_val)
y_p = clf.predict_proba(X_val)
print(classification_report(y_val,y_pred))
from sklearn import metrics
fpr, tpr, thresholds = metrics.roc_curve(y_val,y_p[:,[1]])
metrics.auc(fpr, tpr)
y_test_pred = clf.predict(X_test)
submission = pd.DataFrame({'id':test_id,'TARGET_5Yrs': y_test_pred})
submission.to_csv('mark_submission.csv', index = False)<jupyter_output><empty_output><jupyter_text>SubmissionsTo submit, you can submit from the command line:```bashkaggle competitions submit -c uts-advdsi-nba-career-prediction -f submission.csv -m "Message"```  Tuning<jupyter_code>pipe = Pipeline([
        ('classifier', RandomForestClassifier(oob_score = True))
])

param_dist = {
    'classifier__n_estimators': stats.randint(150, 1000),
    'classifier__max_depth': [3, 4, 5, 6, 7, 8, 9],
    'classifier__max_features': ["auto",'sqrt','log2'],
    'classifier__class_weight': ['balanced','balanced_subsample']
}

cv = RandomizedSearchCV(
    estimator=pipe,
    param_distributions=param_dist,
    random_state=42,
    n_iter=15,
    cv=5,
    n_jobs=7,
    verbose=10
)

cv.fit(X_train, y_train)
preds = cv.predict(X_val)
probs = cv.predict_proba(X_val)
fpr, tpr, thresholds = roc_curve(y_val, probs[:, 1])
roc_auc_score(y_val, probs[:, 1])
confusion_matrix(y_val, preds)
from kaggle.api.kaggle_api_extended import KaggleApi
from dotenv import find_dotenv, load_dotenv
load_dotenv(find_dotenv())
api = KaggleApi()
api.authenticate()
preds = cv.predict(X_test)
pred_name = 'TARGET_5Yrs'
pred_path = processed_data_dir / 'preds_rf_cv.csv'

submission = pd.DataFrame({'id':test_id,'TARGET_5Yrs': preds})

submission.to_csv(pred_path, index = False)

api.competition_submit(file_name=pred_path,
                       message="including randomised CV search",
                       competition=competition,
                       quiet=False)
now()
str(clf)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09724506578947369, 'fraction_numerical': 0.007401315789473684, 'mean_word_length': 4.976658476658477, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '566275', 'license': 'MIT', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1616, 'n_words': 336, 'path': 'notebooks/exploratory/mark_exploration_w1.ipynb', 'repo_name': 'roger-yu-ds/assignment_1', 'size': '4864'}