<jupyter_start><jupyter_text>1. Importing Packages<jupyter_code>import warnings
warnings.filterwarnings('ignore')

import sklearn
from sklearn.tree import export_graphviz
# from sklearn.externals.six import StringIO
from six import StringIO
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer, SimpleImputer
from sklearn.metrics import accuracy_score
import itertools
import pydotplus
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from IPython.display import Image 
import shap
from util import load_data, cindex

import eli5
from eli5.sklearn import PermutationImportance

from pdpbox import pdp, get_dataset, info_plots

from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.naive_bayes import GaussianNB

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report, make_scorer
from scikitplot.metrics import plot_confusion_matrix

from imblearn.under_sampling import NearMiss
from imblearn.over_sampling import SMOTE, SMOTENC

from sklearn.ensemble import BaggingClassifier, AdaBoostClassifier

from sklearn import decomposition
from sklearn.decomposition import PCA
from sklearn.pipeline import make_pipeline<jupyter_output><empty_output><jupyter_text>2. Load & Explore Data loading the data<jupyter_code>X_dev, X_test, y_dev, y_test = load_data(10)<jupyter_output><empty_output><jupyter_text>concatenate the test and train sets into one large dataframe for preprocessing<jupyter_code>df = pd.concat([X_dev, X_test], axis=0)
y = pd.concat([y_dev, y_test], axis=0)
y[y == False] = 0
y[y == True] = 1
df['y'] = y
df['Sex'] = df['Sex'] - 1
df.info()
df.head(10)<jupyter_output><empty_output><jupyter_text>3. Preprocessing 3.1 Handling missing Data<jupyter_code>nulls = (df.isnull().sum() / len(df)) * 100
nulls_df = pd.DataFrame({'columns': nulls.index, 'perc.': nulls.values})
sns.barplot(x = 'perc.', y = 'columns', data = nulls_df)<jupyter_output><empty_output><jupyter_text>we drop ```Systolic BP``` feature as it contains above 50% nulls<jupyter_code>df = df.drop(columns = ['Systolic BP'])
X = df.iloc[:, :-1]
y = df.iloc[:, -1]<jupyter_output><empty_output><jupyter_text>3.2 Balancing the datawe will use **UpSampling** for our data<jupyter_code>print("Before UpSampling:")
print("Number of True outputs : {}" .format(len(y[y==False]))) 
print("Number of False outputs: {}" .format(len(y[y==True])))
y = y.astype('int')
smote = SMOTENC(categorical_features=[3, 11], random_state=0) 
X_US, y_US = smote.fit_resample(X, y)
print("After UpSampling:")
print("Number of True outputs : {}" .format(len(y_US[y_US==False]))) 
print("Number of False outputs: {}" .format(len(y_US[y_US==True])))<jupyter_output>After UpSampling:
Number of True outputs : 6954
Number of False outputs: 6954<jupyter_text>3.3 Scaling/Normalizing the data<jupyter_code>#feature Scaling  
from sklearn.preprocessing import StandardScaler
X_sc = StandardScaler().fit_transform(X_US)<jupyter_output><empty_output><jupyter_text>4. Evaluation Metric   Concordance index**The concordance index (c-index):** is a metric to evaluate the predictions made by an algorithm. It is defined as the proportion of concordant pairs divided by the total number of possible evaluation pairs.$$ \textrm{c-index}= \frac{\textrm{the proportion of concordant pairs } + 0.5  \textrm{ x number of ties}}{\textrm{total number of possible evaluation pairs}} $$> the concordance index is interested on the order of the predictions, not the predictions themselves. >>This is very different from other evaluation measures such as mean squared error or mean absolute error. `c-index characteristics`- A function of how well the ordered values of the continuous predictor correlate to the corresponding event status.- For an increasing events status any strictly increasing predictions the concordance index will still be equal to 1.- The concordance index will be zero for every strictly decreasing predictions.- In case of Ties in your predictions they are counted as half concordant pairs.-  The concordance index supports right censoringwhich means that the event of interest has only occurred for a subset of the observations (e.g., `death of a patient`,  `customer unsubscribe` or  `any time to event model`). - **Random predictions give a mean concordance index of approximately 0.5. So basically a model that hasn’t learned anything give a mean concordance index of 0.5.**  We Will now See The Effect of Preprocessing on c-index with diffrent approaches1. data with missing rows dropped ( >50% of rows) 2. data after dropping the `systolic BP` column & Balancing with UpSampling. 3. data after imputation (mean imputation)  4.1 data with missing rows dropped ( >50% of rows dropped)<jupyter_code># lets load the data again and drop the rows with missing values.
X_dev_np, X_test_np, y_dev_np, y_test_np = load_data(10) # X_dev_np  (np-> no preprocessing) 
X_train_np, X_val_np, y_train_np, y_val_np = train_test_split(X_dev_np, y_dev_np, test_size=0.25, random_state=10)

def fraction_rows_missing(df):
    return sum(df.isnull().any(axis=1)) / len(df)
print(f"Fraction of rows missing from X_train: {fraction_rows_missing(X_train_np):.3f}")
print(f"Fraction of rows missing from X_val: {fraction_rows_missing(X_val_np):.3f}")
print(f"Fraction of rows missing from X_test: {fraction_rows_missing(X_test_np):.3f}")<jupyter_output>Fraction of rows missing from X_train: 0.699
Fraction of rows missing from X_val: 0.704
Fraction of rows missing from X_test: 0.000<jupyter_text>We see that our train and validation sets have missing values, but our test set has no missing values.we will begin with a `complete case analysis`dropping all of the rows with any missing data. Run the following cell to drop these rows from our train and validation sets.<jupyter_code>X_train_dropped = X_train_np.dropna(axis='rows')
y_train_dropped = y_train_np.loc[X_train_dropped.index]
X_val_dropped = X_val_np.dropna(axis='rows')
y_val_dropped = y_val_np.loc[X_val_dropped.index]
dt2 = DecisionTreeClassifier(max_depth=None, random_state=10)
dt2.fit(X_train_dropped, y_train_dropped)
y_train_preds2 = dt2.predict_proba(X_train_dropped)[:, 1]
print(f"Train C-Index: {cindex(y_train_dropped.values, y_train_preds2)}")
y_val_preds = dt2.predict_proba(X_val_dropped)[:, 1]
print(f"test C-Index: {cindex(y_val_dropped.values, y_val_preds)}")<jupyter_output>Train C-Index: 1.0
test C-Index: 0.5629321808510638<jupyter_text>4.2 data after dropping the `systolic BP` ( >50% missing) & Balancing with UpSampling.<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X_US, y_US, test_size=0.25, random_state=10)
# fit a decision tree without setting max_depth -> may lead to over fitting 
# we will tune the parameters later in the model section.

dt = DecisionTreeClassifier(max_depth=None, random_state=10)
dt.fit(X_train, y_train)
y_train_preds = dt.predict_proba(X_train)[:, 1]
print(f"Train C-Index: {cindex(y_train.values, y_train_preds)}")
y_test_preds = dt.predict_proba(X_test)[:, 1]
print(f"test C-Index: {cindex(y_test.values, y_test_preds)}")<jupyter_output>Train C-Index: 1.0
test C-Index: 0.7631520429172449<jupyter_text>Results interpretationThe two classfiers above show two Decision Tree models `one: with features after balancing and dropping Systolic BP` and `second: with missing rows dropped ( >50 of rows dropped)`.The models predicts that **the missing rows drop approach achieved c-index score (~ 0.5)**, but **balanced data achieved c-index score (~ 0.7)** and from that we can deduce two things: -  c-index is indeed a very good evaluation metric for `the time to an event models`, as it show the true model performance which in our simple decision tree classfier is random (model learned nothing). c-index is used as the outcome of interest isn't only whether or not an event occurred, but also when that event occurred. - Tree seems to be overfitting for the first model: it fits the training data so well that it doesn't generalize to other samples such as the validation set.> **We may now proceed with the balancing approach and droping the `Systolic BP` feature, but to make sure we may try mean imputation approach to see how our model would generalize with avoiding `complete case analysis`**> **We will also try different classfiers and use Grid Search for Hyperparameter selection for our best 3 classfiers to improve the c-index score. See Candidate Classification Section**  4.3 data after imputation (mean imputation)We will try to improve the `test C-Index` by filling the missing values and explore if the data is missing at random or not (as we threw away more than half of the data for systolic BP instead of imputing). We will plot histograms of `the dropped rows` against each of the features and compare these to the histograms of the feature in `the entire dataset`. Try to see if one of the features has a signficantly different distribution in the two subsets.<jupyter_code>dropped_rows = X_train_np[X_train_np.isnull().any(axis=1)]

columns_except_Systolic_BP = [col for col in X_train_np.columns if col not in ['Systolic BP']]
some_columns = [col for col in columns_except_Systolic_BP if col in ['Age','Sex', 'Poverty index']]

for col in some_columns:
    sns.distplot(X_train_np.loc[:, col], norm_hist=True, kde=False, label='full data')
    sns.distplot(dropped_rows.loc[:, col], norm_hist=True, kde=False, label='without missing data')
    plt.legend()

    plt.show()<jupyter_output><empty_output><jupyter_text>Most of the features are distributed similarly whether or not we have discarded rows with missing data. In other words missingness of the data is independent of these features.If similar distribution was true across all features, then the data would have been said to be `missing completely at random (MCAR)`.But when considering the age feature, we see that much more data tends to be missing for patients over 65. The reason could be that blood pressure was measured less frequently for old people to avoid placing additional burden on them.As missingness is related to one or more features, the missing data is said to be `missing at random (MAR)` and we will replace the missing values with `mean substitution approach`<jupyter_code># Impute values using the mean
imputer = SimpleImputer(strategy='mean')
imputer.fit(X_train_np)
X_train_mean_imputed = pd.DataFrame(imputer.transform(X_train_np), columns=X_train_np.columns)
X_val_mean_imputed = pd.DataFrame(imputer.transform(X_val_np), columns=X_val_np.columns)
dt = DecisionTreeClassifier(max_depth=None, random_state=10)
dt.fit(X_train_mean_imputed, y_train_np)
y_train_preds = dt.predict_proba(X_train_mean_imputed)[:, 1]
print(f"Train C-Index: {cindex(y_train_np.values, y_train_preds)}")
y_test_preds = dt.predict_proba(X_val_mean_imputed)[:, 1]
print(f"test C-Index: {cindex(y_val_np.values, y_test_preds)}")<jupyter_output>Train C-Index: 1.0
test C-Index: 0.5942460899076951<jupyter_text>We should see that avoiding complete case analysis (dropping rows with missing valuees) allows our model to generalize a bit better.**We Will now proceed with the balanced data and with the systolic BP dropped as it had the best score**  5 Feature Selection1. Permutation Importance2. PCA (Dimentionality reduction) 5.1  Permutation Importance- Feature importance applied after the model has been fitted.- It selects each feature column one at a time and shuffles/permutate that column randomly. Then it calculates how that affect the prediction/accuracy of the model - If changing value of a feature column affect the accuracy of the model heavily then the importance of that column feature is higher. In this way, feature importance is calculated.There are many ways you can calculate feature importance.- we will use `Permutation importance` by the eli5 library.- PCA<jupyter_code># X_dev, X_test, y_dev, y_test = train_test_split(X_US, y_US, test_size=0.20, random_state=10)
# X_train, X_val, y_train, y_val = train_test_split(X_dev, y_dev, test_size=0.25, random_state=10)
# create and fit a dummy model
dt = DecisionTreeClassifier(max_depth=None, random_state=10)
dt.fit(X_train, y_train)

# show importance of the features using permutation importance method
permumtation_impor = PermutationImportance(dt, random_state=2019).fit(X_test, y_test)
eli5.show_weights(permumtation_impor, feature_names = X_test.columns.tolist())<jupyter_output><empty_output><jupyter_text>> From the permutation importance output we can now see the most important features at the top and the less important features is at the bottom.<jupyter_code># try using 5 best features and check model
X_fs = X_US.drop(['Race', 'Serum Magnesium','BMI','Diastolic BP','Serum Protein'], axis = 1)
X_train_fs, X_test_fs, y_train_fs, y_test_fs = train_test_split(X_fs, y_US, test_size=0.25, random_state=10)
dt = DecisionTreeClassifier(max_depth=None, random_state=10)
dt.fit(X_train_fs, y_train_fs)
y_train_preds = dt.predict_proba(X_train_fs)[:, 1]
print(f"Train C-Index: {cindex(y_train_fs.values, y_train_preds)}")
y_test_preds = dt.predict_proba(X_test_fs)[:, 1]
print(f"test C-Index: {cindex(y_test_fs.values, y_test_preds)}")<jupyter_output>Train C-Index: 1.0
test C-Index: 0.7654273557727626<jupyter_text>> **`Note:`even after removing 5 features (least permutation score) the testing c-index doesn't change as much from our best preprocessing technique on all the features.**  5.2 PCA<jupyter_code># lets choose the first 5 principle components and see how
# it is compared to the best 5 features from permutation 
pca = decomposition.PCA(n_components=5) 
pca.fit(X_US)
X = pca.transform(X_US)

pca_lr = make_pipeline(PCA(n_components=10), DecisionTreeClassifier(max_depth=None, random_state=10))
pca_lr.fit(X_train, y_train)
print(f"Train C-Index:{pca_lr.score(X_train, y_train)}" )
print(f"Test C-Index:{pca_lr.score(X_test, y_test)} ")<jupyter_output>Train C-Index:1.0
Test C-Index:0.7144089732528042<jupyter_text>> **After performing the PCA we can notice that even when choosing high number of principal components the model performance wasn't better than removing the least relevant features from the permutation** Now we will fed our models with `X_train_fs`, ` X_test_fs `,` y_train_fs`,` y_test_fs` which was calculated using:>1. Data after dropping the systolic BP ( >50% missing) & Balancing with UpSampling.>2. Feature Selection using permutation importance.<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X_fs, y_US, test_size=0.25, random_state=10)<jupyter_output><empty_output><jupyter_text>6.How a single feature affect our prediction (PDP)we previously had the most important features of our model. It's a good insight, but we don't know how each feature affecting accuracy (Is sex increasing or decreasing the risk of death?).`Partial Dependance Plot (PDP):` shows how each feature will affect the prediction of the model `increase` or `decrease` - PDP calculated after a model is fitted.- Use single row for prediction. Repeatedly alter a variable value to make a series prediction.- Do that for multiple rows and plot the average prediction on the vertical axes<jupyter_code>def pdpplot( feature_to_plot, pdp_model = dt, pdp_dataset = X_test, pdp_model_features = list(X_test)):
    pdp_risk = pdp.pdp_isolate(model=pdp_model, dataset=pdp_dataset, model_features=pdp_model_features, feature=feature_to_plot)
    fig, axes = pdp.pdp_plot(pdp_risk, feature_to_plot, figsize = (10, 5),plot_params={})   
pdpplot('Sex')
plt.show()<jupyter_output><empty_output><jupyter_text>- The y axis is interpreted as change in the prediction from what it would be predicted at the baseline or leftmost value. A blue shaded area indicates level of confidence (standard deviation).- **PDP(Sex)** The probability of death risk is decreased in `females` [ poor men! :( ].<jupyter_code>pdpplot('Age')
plt.show()<jupyter_output><empty_output><jupyter_text>- **PDP(Age)** The probability of death risk is decreased if the age is over 60<jupyter_code>pdpplot('Pulse pressure')
plt.show()<jupyter_output><empty_output><jupyter_text>- **PDP(Pulse pressure)** In the third plot, we see that death risk probability increases  after pulse pressure hits 80.<jupyter_code>pdpplot('TIBC')
plt.show()<jupyter_output><empty_output><jupyter_text>- **PDP(TIBC)** Lastly, we see that death risk probability increases after TIBC hits 410.  7.Why my model predicted that (SHAP Values)In the previous section, we already have seen how our model works, which features are important and how individuals feature affect the prediction of our model. But now we want to know why an individual prediction is made. Why our model made such a prediction? SHAP Values (an acronym from SHapley Additive exPlanations) break down a prediction to show the impact of each feature. It explains why a model made a certain prediction. Previously, we see the explanation of the model after training and how each feature will affect the prediction of the model. Now we are going to look at the individual prediction that our model made. Meaning we will now look at our model after it has made a prediction.> How much a prediction is driven by the factor of sex or pulse presure instead of some baseline value of them? And we do these for each feature. The rules for calculating SHAP values is:$$ \textrm{sum(SHAP values for all features) = pred_for_patient - pred_for_baseline_values}$$- SHAP: how each feature affected the prediction (after prediction).- It calculates how the value of a feature affected the predicted outcome instead of some baseline value.- It does it for every feature and shows the contribution of each feature for the outcome.- SHAP interpret how feature increase or decrease prediction with reference to the PDP and show how consistent your results.  Now we will make a single prediction for a row and see how the model predicted that<jupyter_code>row_to_show = 20
data_for_prediction = X_test.iloc[row_to_show]  # use 1 row of data here. Could use multiple rows if desired
data_for_prediction_array = data_for_prediction.values.reshape(1, -1)


dt.predict_proba(data_for_prediction_array)<jupyter_output><empty_output><jupyter_text>[1., 0.] : means the subject has death risk after 10 years'**الاعمار بيد الله**<jupyter_code>explainer = shap.TreeExplainer(dt)
# Calculate Shap values
shap_values = explainer.shap_values(data_for_prediction)
shap.initjs()
shap.force_plot(explainer.expected_value[0], shap_values[0], data_for_prediction)
row_to_show2 = 30
data_for_prediction2 = X_test.iloc[row_to_show2]  # use 1 row of data here. Could use multiple rows if desired
data_for_prediction_array2 = data_for_prediction2.values.reshape(1, -1)

explainer = shap.TreeExplainer(dt)
# Calculate Shap values
shap_values = explainer.shap_values(data_for_prediction2)
shap.initjs()
shap.force_plot(explainer.expected_value[0], shap_values[0], data_for_prediction)<jupyter_output><empty_output><jupyter_text>**From the two shap plots ubove we can see that the most important features we get from permutation importance are the most relevant in the prediction and with shap we can see each feature contribution either it increase or decrease the prediction.**  8. Candidate Classification Techniques<jupyter_code>from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, HistGradientBoostingClassifier,\
                            BaggingClassifier, AdaBoostClassifier, VotingClassifier, StackingClassifier
X_train, X_test, y_train, y_test = train_test_split(X_fs, y_US, test_size=0.25, random_state=10)<jupyter_output><empty_output><jupyter_text>8.1 selecting 3 base classifiers<jupyter_code>models = [['LogisticRegression : ', LogisticRegression()],
          ['KNN : ', KNeighborsClassifier()],
          ['SVM : ', SVC()],
          ['DecisionTree : ', DecisionTreeClassifier(random_state=0)],
          ['RandomForest : ', RandomForestClassifier(random_state=0)],
          ['XGBoost : ', XGBClassifier()],
          ['NaiveBayes : ', GaussianNB()],
          ['HistGradientBoosting', HistGradientBoostingClassifier(random_state=0)]]
for name, model in models:
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    print(name)
    y_pred_train = model.predict(X_train)
    print("Training", round(cindex(y_train, y_pred_train),3))
    print("Testing", round(cindex(y_test, y_pred),3))
    print("=============================")<jupyter_output>LogisticRegression : 
Training 0.691
Testing 0.679
=============================
KNN : 
Training 0.856
Testing 0.779
=============================
SVM : 
Training 0.692
Testing 0.683
=============================
DecisionTree : 
Training 1.0
Testing 0.764
=============================
RandomForest : 
Training 1.0
Testing 0.865
=============================
[20:59:17] WARNING: D:\bld\xgboost-split_1643227225381\work\src\learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
XGBoost : 
Training 0.977
Testing 0.879
=============================
NaiveBayes : 
Training 0.697
Testing 0.683
=============================
HistGradientBoosting
Training 0.925
Testing 0.865
=============================<jupyter_text>we will select XGBoost, RandomForest, HistGradientBoost for further hyperparameters tuning and applying ensembling methods on them  8.2 Apply Grid search on the 3 base classifiers<jupyter_code>best_models = { 'HistGradientBoosting': HistGradientBoostingClassifier(random_state=0),
                'RandomForest': RandomForestClassifier(random_state=0),
                'XGBoost': XGBClassifier() }
def print_results(clf):
    print(clf)
    pred = clf.predict(X_train)
    print("Train cindex:", cindex(y_train, pred))
    pred = clf.predict(X_test)
    print("Test cindex:", cindex(y_test, pred))

scorer = make_scorer(cindex)<jupyter_output><empty_output><jupyter_text>Note: Running the cells below may take a few minutes depending on your cpu :)<jupyter_code>parameters_rf = {
            'n_estimators' : [100, 150, 200],
            'max_depth' : [35, 40, 45, 50],
            'max_features' : [0.3, 0.5, 0.7]
}
clf = GridSearchCV(estimator=best_models['RandomForest'], param_grid=parameters_rf, scoring=scorer, n_jobs = -1)
grid_fit = clf.fit(X_train, y_train)
best_models['RandomForest'] = grid_fit.best_estimator_
print_results(best_models['RandomForest'])
parameters_xgb = {
            'n_estimators' : [50, 100],
            'eta' : [0.05, 0.1, 0.3],   #eta: learning_rate
            'gamma' : [0.3, 0.4, 0.5],   #gamma: min_split_loss
            'max_depth' : [10, 15, 20],
            'subsample' : [0.7, 0.9, 1.0],
            'colsample_bytree' : [0.9, 1.0, 1.2]
}
clf = GridSearchCV(estimator=best_models['XGBoost'], param_grid=parameters_xgb, scoring=scorer, n_jobs = -1)
grid_fit = clf.fit(X_train, y_train)
best_models['XGBoost'] = grid_fit.best_estimator_
print_results(best_models['XGBoost'])
parameters_hgb = {
        'max_iter': [300, 400, 500],
        'learning_rate': [0.1, 0.3, 0.5],
        'max_depth' : [15, 20, 25],
        'max_leaf_nodes': [30, 40, 50]
}

clf = GridSearchCV(estimator=best_models['HistGradientBoosting'], param_grid=parameters_hgb, scoring=scorer, n_jobs = -1)
grid_fit = clf.fit(X_train, y_train)
best_models['HistGradientBoosting'] = grid_fit.best_estimator_
print_results(best_models['HistGradientBoosting'])<jupyter_output>HistGradientBoostingClassifier(learning_rate=0.3, max_depth=15, max_iter=500,
                               max_leaf_nodes=50, random_state=0)
Train cindex: 0.987169985287338
Test cindex: 0.869709053799435<jupyter_text>8.3 Voting Ensemble classifier<jupyter_code># apply voting with the best three models to acquire the optimum classifier
optimum_clf = VotingClassifier(
    [('hgb', best_models['HistGradientBoosting']),
     ('xgb', best_models['XGBoost']),
     ('rf', best_models['RandomForest'])],
    voting='soft')

# show final results
print()
optimum_clf.fit(X_train, y_train)
print_results(optimum_clf)<jupyter_output>[21:21:22] WARNING: D:\bld\xgboost-split_1643227225381\work\src\learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
VotingClassifier(estimators=[('hgb',
                              HistGradientBoostingClassifier(learning_rate=0.3,
                                                             max_depth=15,
                                                             max_iter=500,
                                                             max_leaf_nodes=50,
                                                             random_state=0)),
                             ('xgb',
                              XGBClassifier(base_score=0.5, booster='gbtree',
                                            colsample_bylevel=1,
                                            colsample_bynode=1,
                                        [...]
{'fraction_non_alphanumeric': 0.09849964835508322, 'fraction_numerical': 0.0227006329608502, 'mean_word_length': 5.219927095990279, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '471895', 'license': 'MIT', 'n_tokens_mistral': 8449, 'n_tokens_neox': 7910, 'n_words': 2586, 'path': 'phase II.ipynb', 'repo_name': 'GP-TEAM-SBME/10-Year-Risk-of-Death-of-Individuals-from-the-NHANES-I-Epidemiology-Dataset', 'size': '25594'}