<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
from numpy import mean
from numpy import std
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score,train_test_split,GridSearchCV
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
import seaborn as sns
train = pd.read_csv('/content/train_cleaned_and_encoded.csv')
validation = pd.read_csv('/content/validation_scaled_cleaned.csv')
test = pd.read_csv('/content/test_cleaned_scaled.csv')
test.shape
train.shape
validation.shape
train.columns
validation.columns
test.columns
del train['Unnamed: 0']
del test['Unnamed: 0']
del validation['Unnamed: 0']
train.sample(10)
test.sample(10)
validation.sample(10)
cat_cols = [x for x in train.columns if train[x].dtype=="O"]
cat_cols.pop(-2)
cat_cols
meta = {}
for col in cat_cols:
    meta[col] = list(train[col].unique())
for key in meta.keys():
    cats = meta[key]
    codes = [x for x in range(len(cats))]
    d = {n:code for n,code in zip(cats,codes)}
    test[key] = test[key].map(d)
for key in meta.keys():
    cats = meta[key]
    codes = [x for x in range(len(cats))]
    d = {n:code for n,code in zip(cats,codes)}
    train[key] = train[key].map(d)
for key in meta.keys():
    cats = meta[key]
    codes = [x for x in range(len(cats))]
    d = {n:code for n,code in zip(cats,codes)}
    validation[key] = validation[key].map(d)
test.head()
train.head()
validation.head()
test.to_csv('test_cleaned_and_encoded.csv')
plt.boxplot(train['booked_before_n_days'])
train_df = train[train.booked_before_n_days.apply(lambda x:True if x<220 else False)]
plt.boxplot(train_df['booked_before_n_days'])
plt.boxplot(train_df['Room_Rate'])
plt.boxplot(train_df['Discount_Rate'])
sns.histplot(train_df['Discount_Rate'])
from sklearn.preprocessing import StandardScaler,MinMaxScaler
scaler = MinMaxScaler()
scaler.fit_transform(np.array(train_df.Expected_checkin_Year).reshape(-1,1))
sns.histplot(scaler.fit_transform(np.array(train_df.Expected_checkin_Year).reshape(-1,1)))
train_df_bkp = train_df.copy()
train_df_bkp[train_df_bkp.drop(['Reservation_Status','Reservation-id'],1).columns] = scaler.fit_transform(train_df[train_df.drop(['Reservation_Status','Reservation-id'],1).columns])
train_df_bkp
# train_df['Expected_checkin_Year'] = scaler.fit_transform(np.array(train_df.Expected_checkin_Year).reshape(-1,1))
train_df_bkp.shape
train_df_bkp.to_csv('train_cleaned_and_encoded.csv')
validation_bkp = validation.copy()
validation_bkp[validation_bkp.drop(['Reservation_Status','Reservation-id'],1).columns] = scaler.fit_transform(validation_bkp[validation_bkp.drop(['Reservation_Status','Reservation-id'],1).columns])
validation_bkp
validation_bkp.to_csv('validation_scaled_cleaned.csv')
test_bkp = test.copy()
test_bkp.shape
validation_bkp.shape
train_df_bkp.shape
test_bkp[test_bkp.drop(['Reservation-id'],1).columns] = scaler.fit_transform(test_bkp[test_bkp.drop(['Reservation-id'],1).columns])
test_bkp.shape
test_bkp.to_csv('test_cleaned_scaled.csv')
g = sns.countplot(new_train['Reservation_Status'])
# g.set_xticklabels(['Not Fraud','Fraud'])
plt.show()
#Visualization
#Create a new figure and make the size (12,10)
plt.figure(figsize=(12,10))
sns.barplot(x=new_train['Income'], y=new_train['Meal_Type'],palette=sns.cubehelix_palette(len(new_train['Income'])))
# Place the region names at a 90-degree angle.
plt.xticks(rotation= 45)
plt.xlabel('Region')
plt.ylabel('Region Happiness Ratio')
plt.title('Happiness rate for regions')
plt.show()<jupyter_output><empty_output><jupyter_text>Start<jupyter_code>new_train = pd.read_csv('/content/train_cleaned_and_encoded.csv')
del new_train['Unnamed: 0']
from imblearn.over_sampling import RandomOverSampler
ros = RandomOverSampler(random_state=42)
x_ros, y_ros = ros.fit_resample(X, y)
X = new_train.drop(['Reservation_Status','Reservation-id'],1)
y = new_train['Reservation_Status'].map({"Check-In":1,"Canceled":2,"No-Show":3})
pd.DataFrame(x_ros,columns=X.columns)
list(X.columns)
train.isna().sum()
# validate = validation
# validate = validate.drop(['Reservation-id'],1)
X.shape
validation=  pd.read_csv('/content/validation_scaled_cleaned.csv')
del validation['Unnamed: 0']
X_val = validation.drop(['Reservation_Status','Reservation-id'],1)
y_val = validation['Reservation_Status'].map({"Check-In":1,"Canceled":2,"No-Show":3})
X_val.shape
# y_val
import numpy as np
import pandas as pd 
import os

#Import sklearn classes 
from sklearn.model_selection import train_test_split,RepeatedKFold, cross_val_score,KFold, RepeatedStratifiedKFold
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.tree import DecisionTreeClassifier,export_graphviz
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier
from sklearn.dummy import DummyClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
# sklearn utility to compare algorithms
from sklearn import model_selection
!pip install eli5
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
import graphviz 

import warnings  
warnings.filterwarnings('ignore')

from eli5 import explain_weights,show_weights
from yellowbrick import ROCAUC
from yellowbrick.classifier import ClassificationReport
from sklearn.metrics import f1_score
X.shape
pca=PCA(n_components=24)
pca.fit(X)
print('Variance explained by the principal components(in decreasing order): ',pca.explained_variance_ratio_)
print('PCA singular values: ',pca.singular_values_)
X1=pca.transform(X)
print('Shape of transformed X: ',X1.shape)
dum=DummyClassifier(strategy='most_frequent')
dum=dum.fit(X,y)

#compute accuracy
score=dum.score(X_val, y_val)
f1 = f1_score(y_val,dum.predict(X_val),average='macro')
print("Dummy Classifier Accuracy: %.2f%%" % (score * 100.0))
print("Dummy Classifier f1_score: {}".format(f1))
strategy = "most_frequent"
CV_N_REPEATS=20
BINS=10
scores = cross_val_score(dum,X, y, 
                         cv=RepeatedKFold(n_repeats=CV_N_REPEATS), 
                         scoring=None) 
scores_dummy = scores.copy()

score_line = "Scores (Accuracy) mean={0:.2f} +/- {1:.2f} (1 s.d.)".format(scores.mean(),scores.std())
plt.figure(figsize=(7,7))
fig, ax = plt.subplots()
pd.Series(scores).hist(ax=ax, bins=BINS)
ax.set_title(f"RepeatedKFold ({len(scores)} folds) with DummyClassifier({strategy})\n" + score_line);
ax.set_xlabel("Score")
ax.set_ylabel("Frequency")
def plot_tree_graph(model,columns,class_names):
    #This function plots the constructed decision tree
    dot_data = export_graphviz(model,feature_names=columns,class_names=class_names) 
    graph = graphviz.Source(dot_data) 
    return graph


def confusion_mat(y_pred,y_test):
    plt.figure()
    sns.set(font_scale=1.5)
    cm = confusion_matrix(y_pred, y_test)
    sns.heatmap(cm, annot=True, fmt='g')
    plt.title('Confusion matrix', y=1.1)
    plt.ylabel('Actual label')
    plt.xlabel('Predicted label')
    plt.show()
knn=KNeighborsClassifier(n_neighbors=3,n_jobs=10,p=2) 
knn.fit(x_ros,y_ros)

scores = cross_val_score(knn, X_val, y_val, cv=RepeatedStratifiedKFold(n_repeats=CV_N_REPEATS))
print(f"Accuracy mean={scores.mean():0.2f} +/- {scores.std():0.2f} (1 s.d.)")
y_pred_1 = knn.predict(X_val)
print(f1_score(y_val,y_pred_1,average='macro'))
confusion_mat(y_pred_1,y_val)
for depth in [n for n in range(10,21,1)]:
  dt=DecisionTreeClassifier(random_state=1, max_depth=depth)
  dt=dt.fit(x_ros,y_ros)
  dt_scores = cross_val_score(dt, X_val, y_val, cv=RepeatedStratifiedKFold(n_repeats=CV_N_REPEATS))
  y_pred_dt = dt.predict(X_val)
  print(depth,f"Accuracy mean={dt_scores.mean():0.2f} +/- {dt_scores.std():0.2f} (1 s.d.)",f1_score(y_val,y_pred_dt,average='macro'))
dt=DecisionTreeClassifier(random_state=1, max_depth=15)
dt=dt.fit(x_ros,y_ros)
dt_scores = cross_val_score(dt, X_val, y_val, cv=RepeatedStratifiedKFold(n_repeats=CV_N_REPEATS))
y_pred_dt = dt.predict(X_val)
print(depth,f"Accuracy mean={dt_scores.mean():0.2f} +/- {dt_scores.std():0.2f} (1 s.d.)",f1_score(y_val,y_pred_dt,average='macro'))
y_pred_dt2 = dt.predict(X_val)
f1_score(y_val,y_pred_dt2,average='macro')
confusion_mat(y_pred_dt2, y_val)
plt.figure()
graph=plot_tree_graph(dt,X.columns,class_names=['1','2','3'])
bag=BaggingClassifier(n_estimators=20,oob_score=True,max_samples=10,max_features=10) 
bag=bag.fit(x_ros,y_ros)

bag_scores = cross_val_score(bag, X_val, y_val, cv=RepeatedStratifiedKFold(n_repeats=CV_N_REPEATS))
print("Accuracy mean={0:0.2f} +/- {1:0.2f} (1 s.d.)".format(scores.mean(),scores.std()))
print("Out of bag score: {0:0.2f}".format(bag.oob_score_*100) );
from sklearn.metrics import roc_auc_score
sample_leaf_options = [10,20,30,25,35,40]

for leaf_size in sample_leaf_options :
    modelrf = RandomForestClassifier(n_estimators = 200, n_jobs = -1,random_state =50, min_samples_leaf = leaf_size)
    modelrf.fit(x_ros,y_ros)
    print("\n Leaf size :", leaf_size)
    print ("AUC - ROC : ", accuracy_score(y_val,modelrf.predict(X_val)))
    print(f1_score(y_val,modelrf.predict(X_val),average='macro'))
feature_names=X.columns.values
show_weights(modelrf,feature_names=feature_names)
modelgb = GradientBoostingClassifier(n_estimators=100,max_depth=32)
modelgb.fit(x_ros,y_ros)
print(depth)
print(accuracy_score(y_val,modelgb.predict(X_val)))
print(f1_score(y_val,modelgb.predict(X_val),average='macro'))
confusion_mat(modelgb.predict(X_val),y_val)
from keras.layers import Dense,Dropout
from keras.models import Sequential
import tensorflow as tf
Reservation_Status_checkIn = list(train_df_bkp[train_df_bkp['Reservation_Status']=='Check-In'].index)
len(Reservation_Status_checkIn)
import random
filtered_idx = random.sample(Reservation_Status_checkIn,6000)
train_sliced = train_df_bkp.loc[filtered_idx]
no_show = train_df_bkp[train_df_bkp['Reservation_Status']=='No-Show']
canceled = train_df_bkp[train_df_bkp['Reservation_Status']=='Canceled']
new_train = pd.concat([train_sliced, no_show, canceled])
new_train
new_train.to_csv('train_slised.csv')

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV
# Create param grid.   
param_rf=[{'n_estimators' : list(range(10,150,15)),
            'max_features' : list(range(6,32,5)),
            'max_depth' : list(range(2,36,2)),
           'min_samples_split':list(range(1,6,1)),
           'min_samples_leaf':list(range(1,6,1)), 
           'min_weight_fraction_leaf':list(range(0,5,1))
            }]
# Create grid search object
clfrfgs = RandomizedSearchCV(RandomForestClassifier(oob_score=True), param_distributions = param_rf, n_iter=50, cv = 5, refit=True,verbose=1, n_jobs=-1)
# Fit on data
best_clf = clfrfgs.fit(x_ros, y_ros)
print(best_clf.best_params_)
best_clf.best_score_
print(f1_score(y_val,best_clf.predict(X_val),average='macro'))
import re

regex = re.compile(r"\[|\]|<", re.IGNORECASE)

X.columns = [regex.sub("_", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in X.columns.values]
X_val.columns = [regex.sub("_", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in X_val.columns.values]
# X_test.columns = [regex.sub("_", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in X_train.columns.values]
from xgboost import XGBClassifier, plot_importance,to_graphviz

# fit model on training data
param = {'max_depth': 20, 'eta': 0.8,"tree_method":"gpu_hist", 'subsample':1,'learning_rate':0.01, 'n_estimators':1000,'reg_lambda':0.1}
xgb = XGBClassifier(**param)
xgb.fit(x_ros, y_ros)
y_pred = xgb.predict(X_val)

# evaluate predictions
xgb_score = accuracy_score(y_val, y_pred)
print("Accuracy of XGB Classifier: {0:0.2f}".format(xgb_score * 100.0));

#Plot the confusion matrix
confusion_mat(y_pred, y_val)
param_test1 = {
 'max_depth':range(3,10,2),
 'min_child_weight':range(1,6,2)
}
gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=5,min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,objective= 'multi:softmax', nthread=4, scale_pos_weight=1, seed=27), 
param_grid = param_test1,n_jobs=4,iid=False, cv=5)
gsearch1.fit(X,y)
gsearch1.cv_results_
gsearch1.best_score_
gsearch1.best_params_
param_test2 = {
 'max_depth':[3,5,7,9],
 'min_child_weight':[1,3,5]
}
gsearch2 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=5,min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,objective= 'multi:softmax', nthread=4, scale_pos_weight=1, seed=27), 
param_grid = param_test2,n_jobs=4,iid=False, cv=5)
gsearch2.fit(X,y)
gsearch2.best_params_
gsearch2.best_score_
param_test3 = {
 'gamma':[i/10.0 for i in range(0,5)]
}
gsearch3 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=3,min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,objective= 'multi:softmax', nthread=4, scale_pos_weight=1,seed=27), 
param_grid = param_test3,n_jobs=4,iid=False, cv=5)
gsearch3.fit(X,y)
gsearch3.best_score_
gsearch3.best_params_
param_test4 = {
 'subsample':[i/10.0 for i in range(6,10)],
 'colsample_bytree':[i/10.0 for i in range(6,10)]
}
gsearch4 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=140, max_depth=3,min_child_weight=1, gamma=0.2, subsample=0.8, colsample_bytree=0.8,objective= 'multi:softmax', nthread=4, scale_pos_weight=1,seed=27), 
param_grid = param_test4,n_jobs=4,iid=False, cv=5)
gsearch4.fit(X,y)
gsearch4.best_score_
gsearch4.best_params_
param_test5 = {
 'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100]
}
gsearch5 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=177, max_depth=3,
 min_child_weight=1, gamma=0.3, subsample=0.6, colsample_bytree=0.7,
 objective= 'multi:softmax', nthread=4, scale_pos_weight=1,seed=27), 
 param_grid = param_test5,n_jobs=4,iid=False, cv=5)
gsearch5.fit(X,y)
gsearch5.best_score_
gsearch5.best_params_
param_test6 = {
 'learning_rate':[0.1, 0.01, 0.05, 0.02]
}
gsearch6 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=177, max_depth=3,
 min_child_weight=1, gamma=0.2, subsample=0.6, colsample_bytree=0.7,reg_alpha=100,objective= 'multi:softmax', nthread=4, scale_pos_weight=1,seed=27), 
 param_grid = param_test6,n_jobs=4,iid=False, cv=5)
gsearch6.fit(X,y)
gsearch6.best_score_
gsearch6.best_params_
gb_final = XGBClassifier( learning_rate =0.01,
                         n_estimators=1000,
                         max_depth=3,
                         min_child_weight=1,
                         gamma=0.2,
                         subsample=0.6,
                         colsample_bytree=0.7,
                         reg_alpha=0.1, 
                         tree_method = "gpu_hist",
                         nthread=4, 
                         scale_pos_weight=1,
                         seed=27).fit(X,y)
f1_score(y_val,xgb.predict(X_val),average='macro')
f1_score(y_val,gb_final.predict(X_val),average='macro')
# gb_final.predict(X_test)


# f1_score(y_val,y_pred,average='macro')
X.columns = [regex.sub("_", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in X.columns.values]
from sklearn.ensemble import VotingClassifier

ensemble_knn_rf_xgb=VotingClassifier(estimators= [('bag',bag),('Random Forest', dt)], voting='hard')
ensemble_knn_rf_xgb.fit(X,y)

#compute accuracy
print('The ensembled model with all the 3 classifiers is:',ensemble_knn_rf_xgb.score(X_val,y_val))

#make predictions
y_pred = ensemble_knn_rf_xgb.predict(X_val)
#Plot the confusion matrix
confusion_mat(y_pred, y_val)
from sklearn.metrics import f1_score

f1_score(y_val, ensemble_knn_rf_xgb.predict(X_val),average='macro')<jupyter_output><empty_output><jupyter_text>The best model so far is DT<jupyter_code>dt.score(X_val,y_val)
print(classification_report(y_val,dt.predict(X_val)))
dt
import pickle
pickle.dump(dt, open('dt_with_36_f1_score', 'wb'))
dt_with_36_f1_score = pickle.load(open('/content/dt_with_36_f1_score', 'rb'))
dt_with_36_f1_score.predict(X_val)
test = pd.read_csv('/content/test_cleaned_scaled.csv')
test.head()
dt.n_features_
X.columns
test = pd.read_csv('/content/test_cleaned_scaled.csv')
test.head()
test = test.drop(['Reservation-id','Unnamed: 0'],1)
dt_predictions = dt.predict(test)
pd.DataFrame({"Reservation-id":test['Reservation-id'],"Reservation_Status":dt_predictions}).to_csv('submission3.csv')<jupyter_output><empty_output><jupyter_text>Ensemble Learning<jupyter_code>import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import MinMaxScaler
train.head()
train.Expected_checkin_Year = scaler.inverse_transform(train.Expected_checkin_Year)
train.shape
list(train.columns)
list(validation.columns)
validation.shape
train.Expected_checkin_Year = train.Expected_checkin_Year.apply(lambda yr:np.mod(yr,2000))
validation.Expected_checkin_Year = validation.Expected_checkin_Year.apply(lambda yr:np.mod(yr,2000))
values_train = train.drop(['Reservation_Status'],1).values
values_validate = validation.drop(['Reservation_Status','Reservation-id','isWeekend_trip'],1).values
imputer = SimpleImputer()
values_validate.shape
values_train.shape
imputedData_train = imputer.fit_transform(values_train)
imputedData_validation = imputer.fit_transform(values_validate)
imputedData_train
imputedData_validation
scaler2 = MinMaxScaler(feature_range=(0, 1))
normalizedData_train = scaler2.fit_transform(imputedData_train)
normalizedData_validation = scaler2.fit_transform(imputedData_validation)
from sklearn import model_selection
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
y_stk = train['Reservation_Status'].map({"Check-In":1,"Canceled":2,"No-Show":3})
y_stk_val = validation['Reservation_Status'].map({"Check-In":1,"Canceled":2,"No-Show":3})
y_stk_val = np.array(y_stk_val)
y_stk = np.array(y_stk)
X_stk = normalizedData_train
X_stk_val = normalizedData_validation
kfold = model_selection.KFold(n_splits=10, random_state=7)
cart = DecisionTreeClassifier()
num_trees = 200
model_bagg = BaggingClassifier(base_estimator=cart, n_jobs =10,n_estimators=num_trees, random_state=7)
results = model_selection.cross_val_score(model_bagg, X, y, cv=kfold)
print(results.mean())
model_bagg.fit(X,y)
predicted_bagg= model_bagg.predict(X_val)
f1_score(y_val,predicted_bagg,average='macro')
from sklearn.ensemble import AdaBoostClassifier
seed = 7
num_trees = 100
kfold = model_selection.KFold(n_splits=20, random_state=seed)
model_ada = AdaBoostClassifier(n_estimators=num_trees,learning_rate =0.01, random_state=seed)
results = model_selection.cross_val_score(model_ada, X, y, cv=kfold)
print(results.mean())
model_ada.fit(X,y)
predicted_ada = model_ada.predict(X_val)
f1_score(y_val,predicted_ada,average='macro')
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import VotingClassifier

kfold = model_selection.KFold(n_splits=10, random_state=seed)
# create the sub models
estimators = []
model1 = LogisticRegression()
estimators.append(('logistic', model1))
model2 = DecisionTreeClassifier(max_depth=30)
estimators.append(('cart', model2))
model3 = SVC()
estimators.append(('svm', model3))
# create the ensemble model
ensemble = VotingClassifier(estimators)
results = model_selection.cross_val_score(ensemble, X, y, cv=kfold)
print(results.mean())
ensemble.fit(X,y)
X_stk_val
predicted_ens = ensemble.predict(X_val)
ensemble.score(X_val,y_val)
y_stk_val.shape
predicted_ens.shape
f1_score(y_val,predicted_ens,average='macro')
results



from sklearn.datasets import make_blobs
from sklearn.naive_bayes import GaussianNB
model = GaussianNB()
model.fit(X,y)
model.score(X_val,y_val)
f1_score(y_val,model.predict(X_val),average='macro')


# use mlp for prediction on multi-label classification
from numpy import asarray
from sklearn.datasets import make_multilabel_classification
from keras.models import Sequential
from keras.layers import Dense

def get_model(n_inputs, n_outputs):
	model = Sequential()
	model.add(Dense(20, input_dim=n_inputs, kernel_initializer='he_uniform', activation='relu'))
	model.add(Dense(n_outputs, activation='sigmoid'))
	model.compile(loss='binary_crossentropy', optimizer='adam')
	return model

n_inputs, n_outputs = X.shape[0], 3
# get model
model = get_model(n_inputs, n_outputs)
# fit the model on all data
model.fit(np.array(X).reshape(1,-1), np.array(y).reshape(1,-1), verbose=0, epochs=100)
y.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1334530504229479, 'fraction_numerical': 0.02509333207315344, 'mean_word_length': 6.399300699300699, 'pattern_counts': {'":': 15, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '510346', 'license': 'MIT', 'n_tokens_mistral': 8399, 'n_tokens_neox': 8061, 'n_words': 1273, 'path': 'Untitled42.ipynb', 'repo_name': 'Ransaka/Angular2-RecordRTC', 'size': '21161'}