<jupyter_start><jupyter_text>Create a function to fit a base model on `K-1` folds, predict on `1` fold<jupyter_code># This function takes one model and fit it to the train and test data
# It returns the model MASE, CV prediction, and test prediction
def base_fit(model, folds, features, target, trainData, testData):
    # Initialize empty lists and matrix to store data
    model_mase = []
    model_val_predictions = np.empty((trainData.shape[0], 1))
    k=0
    # Loop through the index in KFolds
    model_test_predictions=np.zeros((testData.shape[0],))
    model_val_true=np.zeros((trainData.shape[0],1))

    for train_index, val_index in folds.split(trainData):
        k=k+1
        # Split the train data into train and validation data
        train, validation = trainData.iloc[train_index], trainData.iloc[val_index]
        # Get the features and target
        train_features, train_target = train[features], train[target]
        validation_features, validation_target = validation[features], validation[target]
        
        # Fit the base model to the train data and make prediciton for validation data
        if (model.__class__ == xgb.sklearn.XGBRegressor) | (model.__class__ == lgb.sklearn.LGBMRegressor): 
            print('Fitting a boost model with limited tree rounds')
            evalset = [(validation_features,np.ravel(validation_target))]
            model.fit(train_features, np.ravel(train_target),eval_set =evalset,early_stopping_rounds = 20,verbose = False)
        else:
            model.fit(train_features, train_target.values)
        
        if (model.__class__ == xgb.sklearn.XGBRegressor):
            print(model.best_ntree_limit)
            print('Using xgboost with limited tree rounds')
            validation_predictions = model.predict(validation_features,ntree_limit = model.best_ntree_limit)

        elif (model.__class__ == lgb.sklearn.LGBMRegressor):
            print(model.best_iteration_)
            print('Using lgbmboost with limited tree rounds')
            validation_predictions = model.predict(validation_features,num_iteration = model.best_iteration_)
        else:
            print('Using generic predict')
            validation_predictions = model.predict(validation_features)
        
        
        # Calculate and store the MASE for validation data
        print(mase(validation_predictions,validation_target))
        #model_mase.append(mase(validation_predictions,validation_target))
        
        # Save the validation prediction for level 1 model training
        model_val_predictions[val_index, 0] = validation_predictions.reshape(validation.shape[0])
        model_val_true[val_index,0] = validation_target.values
        model_test_predictions += model.predict(testData[features])    
        
    model_test_predictions = model_test_predictions/k
    # Fit the base model to the whole training data
    #model.fit(trainData[features], np.ravel(trainData[target]))
    # Get base model prediction for the test data
    #model_test_predictions = model.predict(testData[features])
    # Calculate and store the MASE for validation data
        
    #model_val_predictions = model_val_predictions
    model_mase.append(mase(model_val_predictions,model_val_true))
    
    return(model_mase, model_val_predictions, model_test_predictions)<jupyter_output><empty_output><jupyter_text>Create a function to fit a dictionary of models, and get their OOF predictions from the training data<jupyter_code># Function that takes a dictionary of models and fits it to the data using baseFit
# The results of the models are then aggregated and returned for level 1 model training
def stacks(level0_models, folds, features, target, trainData, testData):
    num_models = len(level0_models.keys()) #Number of models
    
    # Initialize empty lists and matrix
    level0_trainFeatures = np.empty((trainData.shape[0], num_models))
    level0_testFeatures = np.empty((testData.shape[0], num_models))
    
    # Loop through the models
    for i, key in enumerate(level0_models.keys()):
        print('Fitting %s -----------------------' % (key))
        model_mase, val_predictions, test_predictions = base_fit(level0_models[key], folds, features, target, trainData, testData)
        
        # Print the average MASE for the model
        print('%s average MASE: %s' % (key, np.mean(model_mase)))
        print('\n')
        
        # Aggregate the base model validation and test data predictions
        level0_trainFeatures[:, i] = val_predictions.reshape(trainData.shape[0])
        level0_testFeatures[:, i] = test_predictions.reshape(testData.shape[0])
        
    return(level0_trainFeatures, level0_testFeatures)<jupyter_output><empty_output><jupyter_text>Create a function that trains a dictionary of stackers<jupyter_code># Function that takes a dictionary of classifiers and train them on base model predictions
def stackerTraining(stacker, folds, level0_trainFeatures, level0_testFeatures, trainData,target = None):
    for k in stacker.keys():
        print('Training stacker %s' % (k))
        stacker_model = stacker[k]
        stacker_mase = []
        y_pred = np.zeros_like(trainData[target].values)
        y_true =  np.zeros_like(trainData[target].values)
        for t, v in folds.split(X, y):
            train, validation = level0_trainFeatures[t,:], level0_trainFeatures[v,:]
            # Get the features and target
            train_features, train_target = train, trainData.iloc[t][target]
            validation_features, validation_target = validation, trainData.iloc[v][target]
            
            
            if (stacker_model.__class__ == xgb.sklearn.XGBRegressor) | (stacker_model.__class__ == lgb.sklearn.LGBMRegressor): 
                print('Fitting a boost model with limited tree rounds')
                evalset = [(validation_features,np.ravel(validation_target))]
                stacker_model.fit(train_features, np.ravel(train_target),eval_set =evalset,early_stopping_rounds = 20,verbose = False)
                print(stacker_model.best_iteration_)
            else:
                stacker_model.fit(level0_trainFeatures[t,:], train_target)
                
            y_pred[v] = stacker_model.predict(level0_trainFeatures[v])
            y_true[v] = trainData.iloc[v][target].values
        
        stacker_mase =mase(y_pred,y_true)
        average_mase = mase(level0_trainFeatures.mean(axis=1),y_true)
        print('%s Stacker MASE: %s' % (k, stacker_mase))
        print('%s Averaging MASE: %s' % (k, average_mase))

        
        
# Get the K fold indexes
n_folds = 6
kf = KFold(n_splits=n_folds, shuffle=False, random_state=156)<jupyter_output><empty_output><jupyter_text>Set up the dictionaries of level 0 and level 1 models<jupyter_code># A dictionary of base models
scaler = make_pipeline(QuantileTransformer(output_distribution='normal'))
level0_models = {}
#level0_models['Lasso'] = make_pipeline(scaler, Lasso(alpha =0.005, random_state=1,max_iter = 2000))
#level0_models['ElasticNet'] = make_pipeline(scaler,ElasticNet(alpha = 0.001))
# level0_models['XGB_rougher_base_a'] = xgb.XGBRegressor(learning_rate=0.05,
#                             n_estimators=400,**{'max_depth': 7, 'gamma': '44.954', 'colsample_bytree': '0.395', 'subsample': '0.993', 'min_child_weight': '133.132'},
#                             silent=1,
#                              random_state =48, nthread = -1)

# level0_models['XGB_rougher_base_b'] =xgb.XGBRegressor(learning_rate=0.05,
#                             n_estimators=400,**{'max_depth': 7, 'gamma': '44.954', 'colsample_bytree': '0.395', 'subsample': '0.993', 'min_child_weight': '133.132'},
#                             silent=1,
#                              random_state =38, nthread = -1)

# # level0_models['XGB_rougher_base_c'] = xgb.XGBRegressor(learning_rate=0.05,
# #                             n_estimators=400,**{'max_depth': 4, 'gamma': '5.102', 'colsample_bytree': '0.344', 'subsample': '0.884', 'min_child_weight': '9.622'},
# #                             silent=1,
# #                              random_state =82, nthread = -1)

# # level0_models['XGB_rougher_base_d'] = xgb.XGBRegressor(learning_rate=0.05,
# #                             n_estimators=400,**{'max_depth': 5, 'gamma': '3.906', 'colsample_bytree': '0.401', 'subsample': '0.760', 'min_child_weight': '9.563'},
# #                             silent=1,
# #                              random_state =11, nthread = -1)


level0_models['LGBM_rougher_base_a'] = lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=500,random_state=91,
                              **{'max_depth': 5, 'num_leaves': 40, 'feature_fraction': '0.453', 'bagging_fraction': '0.276'})

level0_models['LGBM_rougher_base_b'] =lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=500,random_state=92,
                              **{'max_depth': 4, 'num_leaves': 180, 'feature_fraction': '0.322', 'bagging_fraction': '0.150'})
level0_models['LGBM_rougher_base_c'] =lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=500,random_state=93,
                              ** {'max_depth': 5, 'num_leaves': 80, 'feature_fraction': '0.440', 'bagging_fraction': '0.666'})
level0_models['LGBM_rougher_base_d'] =lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=500,random_state=94,
                             **{'max_depth': 6, 'num_leaves': 25, 'feature_fraction': '0.555', 'bagging_fraction': '0.257'})
level0_models['LGBM_rougher_base_e']= lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=500,random_state=7,
                              **{'max_depth': 4, 'num_leaves': 30, 'feature_fraction': '0.658', 'bagging_fraction': '0.863'})




# Train all the base models in the dictionary
level0_trainFeatures_rougher, level0_testFeatures_rougher = stacks(level0_models, kf, X.columns, tgt, train_df, X_test)

# A dictionary of level 1 model to train on base model predictions
stacker = {
            'Enet': ElasticNet(alpha = 0.001),
#           'BR':BayesianRidge(n_iter = int(3e3)),
#           'Ridge1':Ridge(),
            'Ridge50':Ridge(alpha=60),
           'Ridge1e2':Ridge(alpha=1e2),
           'Ridge5e2':Ridge(alpha=5e2),
           'Ridge1e3':Ridge(alpha=1e3),
           'Ridge5e3':Ridge(alpha=5e3),
          'Lasso': Lasso(alpha =0.005, random_state=1,max_iter = 2000),
      'positive_Lasso': Lasso(alpha=0.0001,precompute=True,max_iter=1000, positive=True, random_state=9999, selection='random'),
          'LGBM': lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.07, n_estimators=500,random_state=38,
                              **{'max_depth': 4, 'num_leaves': 32, 'feature_fraction': '0.8', 'bagging_fraction': '0.8'})}

stackerTraining(stacker, kf, level0_trainFeatures_rougher, level0_testFeatures_rougher, train_df,target = tgt)
level1_model = stacker['Enet'].fit(level0_trainFeatures_rougher,train_df[tgt])
level1_test_pred_rougher = level1_model.predict(level0_testFeatures_rougher)
rougher_meta_train = level0_trainFeatures_rougher
rougher_meta_test = level0_testFeatures_rougher

level1_test_pred_rougher.shape<jupyter_output>Training stacker Enet
Enet Stacker MASE: 2.6447062948910194
Enet Averaging MASE: 2.424670570059726
Training stacker Ridge50
Ridge50 Stacker MASE: 2.6445328317688506
Ridge50 Averaging MASE: 2.424670570059726
Training stacker Ridge1e2
Ridge1e2 Stacker MASE: 2.644392545383671
Ridge1e2 Averaging MASE: 2.424670570059726
Training stacker Ridge5e2
Ridge5e2 Stacker MASE: 2.6431340791909035
Ridge5e2 Averaging MASE: 2.424670570059726
Training stacker Ridge1e3
Ridge1e3 Stacker MASE: 2.641835422442308
Ridge1e3 Averaging MASE: 2.424670570059726
Training stacker Ridge5e3
Ridge5e3 Stacker MASE: 2.6362638158379967
Ridge5e3 Averaging MASE: 2.424670570059726
Training stacker Lasso
Lasso Stacker MASE: 2.644493557954057
Lasso Averaging MASE: 2.424670570059726
Training stacker positive_Lasso
positive_Lasso Stacker MASE: 2.633923971881569
positive_Lasso Averaging MASE: 2.424670570059726
Training stacker LGBM
Fitting a boost model with limited tree rounds
24
Fitting a boost model with limited tree rounds
82
[...]<jupyter_text>Let's try to figure out the stacking Level 2<jupyter_code>def solve_pg(A, b, momentum=0.9, maxiter=1000):
    from cvxpy import Variable,Parameter,Minimize,square,norm,Problem
    M, N = A.shape
    x = np.zeros(N)

    AtA = A.T.dot(A)
    Atb = A.T.dot(b)

    stop_count = 0

    # projection helper
    x_ = Variable(N)
    v_ = Parameter(N)
    objective_ =  Minimize(0.5 * norm(x_ - v_, 1))
    constraints_ = [sum(x_) == 1]
    problem_ = Problem(objective_, constraints_)

    def gradient(x):
        return AtA.dot(x) - Atb

    def obj(x):
        return 0.5 * np.linalg.norm(A.dot(x) - b,ord=1)

    it = 0
    while True:
        grad = gradient(x)

        # line search
        alpha = 1
        beta = 0.5
        sigma=1e-2
        old_obj = obj(x)
        while True:
            new_x = x - alpha * grad
            new_obj = obj(new_x)
            if old_obj - new_obj >= sigma * grad.dot(x - new_x):
                break
            else:
                alpha *= beta

        x_old = x[:]
        x = x - alpha*grad

        # projection
        v_.value = x
        problem_.solve()
        x = np.array(x_.value.flat)

        y = x + momentum * (x - x_old)

        if np.abs(old_obj - obj(x)) < 1e-2:
            stop_count += 1
        else:
            stop_count = 0

        if stop_count == 3:
            print('early-stopping @ it: ', it)
            return x

        it += 1

        if it == maxiter:
            return x

x = solve_pg(level0_trainFeatures_rougher, train_df[tgt].values)
print('sum x: ', np.sum(x))
print(mase(level0_trainFeatures_rougher.dot(x),train_df[tgt].values))<jupyter_output>early-stopping @ it:  8
sum x:  0.9999999999997128
2.4246217512197545<jupyter_text>Repeat the procedure with Final<jupyter_code>year = 2019
tgt = 'final.output.recovery'

X = data_dict[year]['X_train_tsclean']
y = data_dict[year]['y_train']
X_test = data_dict[year]['X_test_ts']

mask = data_dict[year]['mask']
exclude_pts = data_dict[year]['excl'].set_index('date').tz_localize('UTC')
#mask_na_two_row=y[y[tgt].isna()].index.union(y[y[tgt].isna()].index + pd.Timedelta('1 hour')).union(y[y[tgt].isna()].index + pd.Timedelta('2 hour'))
inds = mask.index.difference(exclude_pts.index)
print(X.shape)

X=X.loc[inds,:]
y=y.loc[inds,:]
mask=mask[inds]
print(X.shape)

print(f'1) X shape: {X.shape},y: {y.shape}')
X = X[mask]
y = y[mask][tgt]

train_df = pd.concat([X,y],axis= 1)
print(f'1) X shape: {X.shape},y: {y.shape}')
print(X_test.shape)
train_df.head()<jupyter_output>(16859, 313)
(16063, 313)
1) X shape: (16063, 313),y: (16063, 7)
1) X shape: (12838, 313),y: (12838,)
(5856, 313)<jupyter_text>Create dicts of stackers and base models<jupyter_code># A dictionary of base models
scaler = make_pipeline(QuantileTransformer(output_distribution='normal'))
level0_models = {}
level0_models['Lasso'] = make_pipeline(scaler, Lasso(alpha =0.005, random_state=1,max_iter = 2000))
level0_models['ElasticNet'] = make_pipeline(scaler,ElasticNet(alpha = 0.001))
def fair_obj(preds, dtrain):
    """y = c * abs(x) - c**2 * np.log(abs(x)/c + 1)"""
    x = preds - dtrain.get_labels()
    c = 1
    den = abs(x) + c
    grad = c*x / den
    hess = c*c / den ** 2
    return grad, hess

# level0_models['XGB_final_base_a'] = xgb.XGBRegressor(learning_rate=0.05,
#                             n_estimators=400,**{'max_depth': 3, 'gamma': '17.158', 'colsample_bytree': '0.442', 'subsample': '0.644', 'min_child_weight': '9.733'},
#                             silent=1,
#                              random_state =48, nthread = -1)

# level0_models['XGB_final_base_b'] =xgb.XGBRegressor(learning_rate=0.05,
#                             n_estimators=400,**{'max_depth': 4, 'gamma': '18.571', 'colsample_bytree': '0.745', 'subsample': '0.681', 'min_child_weight': '9.024'},
#                             silent=1,
#                              random_state =38, nthread = -1)

# level0_models['XGB_final_base_c'] = xgb.XGBRegressor(learning_rate=0.05,
#                             n_estimators=400,**{'max_depth': 2, 'gamma': '16.491', 'colsample_bytree': '0.522', 'subsample': '0.844', 'min_child_weight': '5.096'},
#                             silent=1,
#                              random_state =82, nthread = -1)

# level0_models['XGB_final_base_d'] = xgb.XGBRegressor(learning_rate=0.05,
#                             n_estimators=400,**{'max_depth': 3, 'gamma': '16.766', 'colsample_bytree': '0.540', 'subsample': '0.774', 'min_child_weight': '12.674'},
#                             silent=1,
#                              random_state =11, nthread = -1)

level0_models['LGBM_final_base_a'] = lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=400,random_state=96,
                              **{'max_depth': 7, 'num_leaves': 165, 'feature_fraction': '0.457', 'bagging_fraction': '0.414'})

level0_models['LGBM_final_base_b'] =lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=400,random_state=973,
                              **{'max_depth': 6, 'num_leaves': 85, 'feature_fraction': '0.493', 'bagging_fraction': '0.505'})

level0_models['LGBM_final_base_c'] =lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=400,random_state=937,
                              **{'max_depth': 8, 'num_leaves': 70, 'feature_fraction': '0.738', 'bagging_fraction': '0.376'})
level0_models['LGBM_final_base_d'] =lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.05, n_estimators=400,random_state=49,
                              **{'max_depth': 3, 'num_leaves': 10, 'feature_fraction': '0.465', 'bagging_fraction': '0.561'})



# level0_models['LGBM_final_base_e'] = lgb.LGBMRegressor(objective=obj,
#                               learning_rate=0.07, n_estimators=500,random_state=7,
#                               **{'max_depth': 4, 'num_leaves': 63, 'feature_fraction': '0.89', 'bagging_fraction': '0.757'})
# level0_models['LGBM_final_base_f'] = lgb.LGBMRegressor(objective=obj,
#                               learning_rate=0.07, n_estimators=500,random_state=8,
#                               **{'max_depth': 4, 'num_leaves': 63, 'feature_fraction': '0.879', 'bagging_fraction': '0.727'})
# level0_models['LGBM_final_base_g'] = lgb.LGBMRegressor(objective=obj,
#                               learning_rate=0.07, n_estimators=500,random_state=9,
#                               **{'max_depth': 5, 'num_leaves': 65, 'feature_fraction': '0.879', 'bagging_fraction': '0.727'})
# level0_models['LGBM_final_base_h'] = lgb.LGBMRegressor(objective=obj,
#                               learning_rate=0.07, n_estimators=500,random_state=10,
#                               **{'max_depth': 4, 'num_leaves': 60, 'feature_fraction': '0.797', 'bagging_fraction': '0.982'})
# level0_models['LGBM_final_base_i'] = lgb.LGBMRegressor(objective=obj,
#                               learning_rate=0.07, n_estimators=500,random_state=12,
#                               **{'max_depth': 5, 'num_leaves': 60, 'feature_fraction': '0.8', 'bagging_fraction': '0.92'})



# #level0_models['KNN_final_a'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 254, 'weights': 'distance', 'leaf_size': 16}))
#level0_models['KNN_final_b'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 50, 'weights': 'distance', 'leaf_size': 18}))
#level0_models['KNN_final_c'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 15, 'weights': 'distance', 'leaf_size': 30.0}))
# level0_models['KNN_final_d'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 5, 'weights': 'uniform', 'leaf_size': 24.0}))
#level0_models['KNN_rougher_b_bray'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 50, 'weights': 'distance','metric':'braycurtis', 'leaf_size': 18}))
#level0_models['KNN_rougher_c_bray'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 15, 'weights': 'distance', 'leaf_size': 30.0,'metric':'braycurtis'}))
# level0_models['KNN_rougher_d_bray'] = make_pipeline(scaler,KNeighborsRegressor(n_jobs = -1,**{'n_neighbors': 5, 'weights': 'uniform', 'leaf_size': 24.0,'metric':'braycurtis'}))



# Train all the base models in the dictionary
level0_trainFeatures_final, level0_testFeatures_final = stacks(level0_models, kf, X.columns, tgt, train_df, X_test)
# A dictionary of level 1 model to train on base model predictions
stacker = {
#            'Enet': ElasticNet(alpha = 0.001),
#           #'RF':RandomForestRegressor(max_depth = 3,n_estimators=200),
#           'BR':BayesianRidge(n_iter = int(3e3)),
#           'Ridge1':Ridge(),
#           'Ridge1e2':Ridge(alpha=1e2),
#           'Ridge5e2':Ridge(alpha=5e2),
#           'Ridge1e3':Ridge(alpha=1e3),
#           'Ridge5e3':Ridge(alpha=5e3),
#           'Lasso': Lasso(alpha =0.005, random_state=1,max_iter = 2000),
          'LGBM': lgb.LGBMRegressor(objective='mae',
                              learning_rate=0.07, n_estimators=500,random_state=40,
                              **{'max_depth': 3, 'num_leaves':8, 'feature_fraction': '0.9', 'bagging_fraction': '0.7'})}

stackerTraining(stacker, kf, level0_trainFeatures_final, level0_testFeatures_final, train_df,target = tgt)
level1_model = stacker['LGBM'].fit(level0_trainFeatures_final,train_df[tgt])
level1_test_pred_final = level1_model.predict(level0_testFeatures_final)
level1_test_pred_final.shape
x<jupyter_output><empty_output><jupyter_text>Make a submission:<jupyter_code>preds = pd.DataFrame(data = {'date':X_test.index,'rougher.output.recovery':level1_test_pred_rougher, 'final.output.recovery':level0_testFeatures_final.mean(axis=1)})
stacked_preds_sub = preds
stacked_preds_sub['date'] = stacked_preds_sub['date'].dt.strftime('%Y-%m-%dT%H:%M:%SZ')
stacked_preds_sub.set_index('date',inplace=True)
#stacked_preds_sub.drop_duplicates(inplace=True)
stacked_preds_sub.to_csv('../results/stacked_sub_lgb_lasso_base_alldata_lagdiff2_nots.csv')
stacked_preds_sub.plot(style=['o','o'],figsize = (20,10),alpha=0.9)

preds_av = pd.DataFrame(data = {'date':X_test.index,'rougher.output.recovery':level0_testFeatures_rougher.mean(axis=1), 'final.output.recovery':level0_testFeatures_final.mean(axis=1)})

preds_av['date'] = preds_av['date'].dt.strftime('%Y-%m-%dT%H:%M:%SZ')
preds_av.set_index('date',inplace=True)
preds_av.plot(figsize = (20,10),style=['o','o'],alpha=0.9)

preds_av.to_csv('../results/stacked_sub_lgb_lasso_base_alldata_lagdiff2_nots_averaged.csv')
r = np.power(level0_testFeatures_rougher.prod(axis=1),1/level0_testFeatures_rougher.shape[1])
f = np.power(level0_testFeatures_final.prod(axis=1),1/level0_testFeatures_final.shape[1])

preds_av = pd.DataFrame(data = {'date':X_test.index,'rougher.output.recovery':r, 'final.output.recovery':f})
preds_av['date'] = preds_av['date'].dt.strftime('%Y-%m-%dT%H:%M:%SZ')
preds_av.set_index('date',inplace=True)
preds_av.plot(figsize = (20,10),style=['o','o'],alpha=0.9)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13561345899611923, 'fraction_numerical': 0.05262484540918589, 'mean_word_length': 3.386457164234942, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '336773', 'license': 'MIT', 'n_tokens_mistral': 8976, 'n_tokens_neox': 8192, 'n_words': 1411, 'path': 'notebooks/stack_models_lagdiffs-4p857.ipynb', 'repo_name': 'AntonBiryukovUofC/unearthed-zinc', 'size': '23449'}