<jupyter_start><jupyter_text>Step1  i) Base Model: Train base_train and test base_test using default params<jupyter_code># Base Model: Train base_train and test base_test using default params
sc=pyspark.SparkContext.getOrCreate()
sqlContext = SQLContext(sc)
sqlContext.clearCache()

trainData = sqlContext.read.csv("base_train.csv", header = True)
testData = sqlContext.read.csv("base_test.csv", header = True)
trainData = trainData.select(*(F.col(c).cast("double").alias(c) for c in trainData.columns))
testData = testData.select(*(F.col(c).cast("double").alias(c) for c in testData.columns))
train_count = trainData.select("Class").count()
train_fraud_count = trainData.select("Class").where(trainData.Class == 1).count()
test_count = testData.select("Class").count()
print("train before sampling: " + str(train_count))
print("fraud ratio: " + str(train_fraud_count/train_count))
print("test: " + str(test_count))

features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']
vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')
df_train = vectorAssembler.transform(trainData)
df_test = vectorAssembler.transform(testData)


lr = LogisticRegression(featuresCol = 'features', labelCol='Class')
# Fit the model
lrModel = lr.fit(df_train)
# predict the model
pred = lrModel.transform(df_test)
pred = pred.select("features", "Class", "rawPrediction", "prediction")

#evaluate
pred.groupby('Class').agg({'Class': 'count'}).show()
pred.groupby('prediction').agg({'prediction': 'count'}).show()
precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()
recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()
f1 = 2 * precision * recall /(precision + recall)


print("Precision is ", precision)
print("Recall is ", recall)
print("F1 score is ", f1)
evaluator = BinaryClassificationEvaluator(
    labelCol="Class", metricName="areaUnderROC")
print('Test Area Under ROC', evaluator.evaluate(pred))<jupyter_output>train before sampling: 181584
fraud ratio: 0.0016961846858754076
test: 45396
+-----+------------+
|Class|count(Class)|
+-----+------------+
|  0.0|       45305|
|  1.0|          91|
+-----+------------+

+----------+-----------------+
|prediction|count(prediction)|
+----------+-----------------+
|       0.0|            45335|
|       1.0|               61|
+----------+-----------------+

Precision is  0.8852459016393442
Recall is  0.5934065934065934
F1 score is  0.7105263157894737
Test Area Under ROC 0.9775965343563068<jupyter_text>Step2. Comparing random and stratified cv  5-fold random cv with default param and sample_method=None<jupyter_code>df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9',
                                            'V10','V11','V12','V13','V14','V15','V16','V17','V18',
                                            'V19','V20','V21','V22','V23','V24','V25','V26','V27',
                                            'V28','Amount']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, False, 'None', 0, False, features, 'lr', [0.5],[100])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
sqlContext.clearCache()<jupyter_output>[0.5, 100]
train: 152562
fraud ratio: 0.0014420366801693738
train: 152482
fraud ratio: 0.0016592122348867407

 precision, recall, f_score: 0.7011494252873564, 0.6931818181818182, 0.6971428571428572

 precision, recall, f_score: 0.9411764705882353, 0.5818181818181818, 0.7191011235955057
train: 152538
fraud ratio: 0.0017372720240202441
train: 152553
fraud ratio: 0.0018223174896593315

 precision, recall, f_score: 1.0, 0.32558139534883723, 0.49122807017543857

 precision, recall, f_score: 0.8823529411764706, 0.5, 0.6382978723404256
train: 152499
fraud ratio: 0.0016655846923586384

 precision, recall, f_score: 0.90625, 0.5370370370370371, 0.6744186046511628
grid search precision: 0.8861857674104124
grid search recall: 0.5275236864771748
grid search f-score: 0.6613579355254386<jupyter_text>5-fold stratified cv with default param and sample_method=None<jupyter_code>#stratified 5cv with default param(threshold version)
df = sqlContext.read.csv("base_train.csv", header = True)
features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'lr', [0.5],[100])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
sqlContext.clearCache()<jupyter_output>[0.5, 100]
train: 145268
fraud ratio: 0.001700305641985847
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.8947368421052632, 0.5573770491803278, 0.686868686868687
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.8823529411764706, 0.4838709677419355, 0.625
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.76, 0.6129032258064516, 0.6785714285714285
train: 145270
fraud ratio: 0.0017002822330832244

 precision, recall, f_score: 0.8292682926829268, 0.5483870967741935, 0.6601941747572816

 precision, recall, f_score: 0.8604651162790697, 0.6065573770491803, 0.7115384615384615
grid search precision: 0.845364638448746
grid search recall: 0.5618191433104178
grid search f-score: 0.6750248874592001<jupyter_text>Step3. stratified cv grid search<jupyter_code>#grid search with threshold [0.2,150] f1=0.73
df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9',
                                            'V10','V11','V12','V13','V14','V15','V16','V17','V18',
                                            'V19','V20','V21','V22','V23','V24','V25','V26','V27',
                                            'V28','Amount']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'lr', [0.2,0.3,0.4],[110,150,200])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
print("grid search hyper:", gs_hyper)
sqlContext.clearCache()<jupyter_output>[0.2, 110]
train: 145268
fraud ratio: 0.001700305641985847
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.7719298245614035, 0.7213114754098361, 0.7457627118644067

 precision, recall, f_score: 0.782608695652174, 0.5806451612903226, 0.6666666666666667
train: 145266
fraud ratio: 0.0016934451282474908
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.8070175438596491, 0.7419354838709677, 0.7731092436974789

 precision, recall, f_score: 0.8235294117647058, 0.6774193548387096, 0.743362831858407
train: 145270
fraud ratio: 0.0017002822330832244

 precision, recall, f_score: 0.8125, 0.639344262295082, 0.7155963302752295
[0.2, 150]
train: 145268
fraud ratio: 0.001700305641985847
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.7586206896551724, 0.7213114754098361, 0.7394957983193277

 precision, recall, f_score: 0.782608695652174, 0.5806451612903226, 0.6666666666666667
train: 145266
fraud ratio: 0.0[...]<jupyter_text>Best hyper parameter in first grid search grid search hyper: [0.2, 150]  use best param in step3 test base_test<jupyter_code>lr = LogisticRegression(featuresCol = 'features', labelCol='Class',threshold=0.2, maxIter=150)
lrModel = lr.fit(df_train)
pred = lrModel.transform(df_test)
pred = pred.select("features", "Class", "rawPrediction", "prediction")
pred.groupby('Class').agg({'Class': 'count'}).show()
pred.groupby('prediction').agg({'prediction': 'count'}).show()
precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()
recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()
f1 = 2 * precision * recall /(precision + recall)
print("Precision is ", precision)
print("Recall is ", recall)
print("F1 score is ", f1)<jupyter_output>+-----+------------+
|Class|count(Class)|
+-----+------------+
|  0.0|       45305|
|  1.0|          91|
+-----+------------+

+----------+-----------------+
|prediction|count(prediction)|
+----------+-----------------+
|       0.0|            45317|
|       1.0|               79|
+----------+-----------------+

Precision is  0.8227848101265823
Recall is  0.7142857142857143
F1 score is  0.7647058823529411<jupyter_text>result after first grid search Precision is  0.8227848101265823 Recall is  0.7142857142857143 F1 score is  0.7647058823529411  Step4. use best param from 3) for feature coefficients<jupyter_code>#feature coefficient
features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9',
                                            'V10','V11','V12','V13','V14','V15','V16','V17','V18',
                                            'V19','V20','V21','V22','V23','V24','V25','V26','V27',
                                            'V28','Amount']
lr = LogisticRegression(featuresCol = 'features', labelCol='Class',threshold=0.2, maxIter=150)
lrModel = lr.fit(df_train)
coefficients = lrModel.coefficients
abs_coefficients=[abs(i) for i in coefficients]
tmp = pd.DataFrame({'Feature': features, 'abs_coefficients': abs_coefficients})
tmp = tmp.sort_values(by='abs_coefficients',ascending=False)
plt.figure(figsize = (15,8))
plt.title('abs_coefficients',fontsize=14)
s = sns.barplot(x='Feature',y='abs_coefficients',data=tmp)
s.set_xticklabels(s.get_xticklabels(),rotation=90)
plt.show()

pred = lrModel.transform(df_test)
pred = pred.select("features", "Class", "rawPrediction", "prediction")
precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()
recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()
f1 = 2 * precision * recall /(precision + recall)
print("Precision is ", precision)
print("Recall is ", recall)
print("F1 score is ", f1)<jupyter_output><empty_output><jupyter_text>Step5. stratified cv using features selected from 4) 'V11', 'V1','V21','V3','V22','V6','V28'<jupyter_code>df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V11', 'V1','V21','V3','V22','V6','V28']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None',0, False, features, 'lr', [0.2],[150])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
print("grid search hyper:", gs_hyper)
sqlContext.clearCache()<jupyter_output>[0.2, 150]
train: 145266
fraud ratio: 0.0016934451282474908
train: 145268
fraud ratio: 0.001700305641985847

 precision, recall, f_score: 0.8205128205128205, 0.5245901639344263, 0.64
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.7608695652173914, 0.5645161290322581, 0.6481481481481481
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.9024390243902439, 0.5967741935483871, 0.7184466019417476
train: 145270
fraud ratio: 0.0017002822330832244

 precision, recall, f_score: 0.84375, 0.43548387096774194, 0.5744680851063829

 precision, recall, f_score: 0.8918918918918919, 0.5409836065573771, 0.673469387755102
grid search precision: 0.8438926604024696
grid search recall: 0.5324695928080381
grid search f-score: 0.6529490041012765
grid search hyper: [0.2, 150]<jupyter_text>Step6. revisit step5 to choose best feature  'V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10'<jupyter_code>df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None',0, False, features, 'lr', [0.2],[150])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
print("grid search hyper:", gs_hyper)
sqlContext.clearCache()<jupyter_output>[0.2, 150]
train: 145266
fraud ratio: 0.0016934451282474908
train: 145268
fraud ratio: 0.001700305641985847

 precision, recall, f_score: 0.9090909090909091, 0.6557377049180327, 0.761904761904762
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.8775510204081632, 0.6935483870967742, 0.7747747747747746
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.8, 0.6451612903225806, 0.7142857142857142
train: 145270
fraud ratio: 0.0017002822330832244

 precision, recall, f_score: 0.8260869565217391, 0.6129032258064516, 0.7037037037037037

 precision, recall, f_score: 0.8333333333333334, 0.5737704918032787, 0.6796116504854368
grid search precision: 0.849212443870829
grid search recall: 0.6362242199894236
grid search f-score: 0.7274487534230674
grid search hyper: [0.2, 150]<jupyter_text>'V11','V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16'<jupyter_code>df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None',0, False, features, 'lr', [0.2],[150])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
sqlContext.clearCache()<jupyter_output>[0.2, 150]
train: 145268
fraud ratio: 0.001700305641985847
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.7818181818181819, 0.6935483870967742, 0.7350427350427351

 precision, recall, f_score: 0.7307692307692307, 0.6229508196721312, 0.672566371681416
train: 145266
fraud ratio: 0.0016934451282474908
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.7352941176470589, 0.8064516129032258, 0.7692307692307693

 precision, recall, f_score: 0.8958333333333334, 0.6935483870967742, 0.7818181818181819
train: 145270
fraud ratio: 0.0017002822330832244

 precision, recall, f_score: 0.8372093023255814, 0.5901639344262295, 0.6923076923076923
grid search precision: 0.7961848331786773
grid search recall: 0.681332628239027
grid search f-score: 0.7342948142665915<jupyter_text>Step7. Using finalized features do stratified cv grid search for marginal improvement finalized feature from step6: 'V11','V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16'<jupyter_code>df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']
gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'lr', [0.1,0.15,0.2],[130,150,170])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
print("grid search hyper:", gs_hyper)
sqlContext.clearCache()<jupyter_output>[0.1, 130]
train: 145266
fraud ratio: 0.0016934451282474908
train: 145268
fraud ratio: 0.001700305641985847

 precision, recall, f_score: 0.7352941176470589, 0.819672131147541, 0.7751937984496124

 precision, recall, f_score: 0.7540983606557377, 0.7419354838709677, 0.7479674796747967
train: 145266
fraud ratio: 0.0016934451282474908
train: 145266
fraud ratio: 0.0016934451282474908

 precision, recall, f_score: 0.7166666666666667, 0.6935483870967742, 0.7049180327868854

 precision, recall, f_score: 0.8148148148148148, 0.7096774193548387, 0.7586206896551724
train: 145270
fraud ratio: 0.0017002822330832244

 precision, recall, f_score: 0.84, 0.6885245901639344, 0.7567567567567568
[0.1, 150]
train: 145266
fraud ratio: 0.0016934451282474908
train: 145268
fraud ratio: 0.001700305641985847

 precision, recall, f_score: 0.7540983606557377, 0.7419354838709677, 0.7479674796747967

 precision, recall, f_score: 0.7575757575757576, 0.819672131147541, 0.7874015748031497
train: 145266
fraud ratio: 0.0[...]<jupyter_text>Best hyper parameter after second grid search grid search hyper: [0.1, 150]  Step8 use stratified cv to compare no sampling, undersampling, and oversampling<jupyter_code>#undersampling with finalized features and best param from step7:[0.1,150]
df = sqlContext.read.csv("base_train.csv", header = True)
features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']
for i in [0.01, 0.05, 0.10, 0.20]:
    gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'under', i, False, features, 'lr', [0.1],[150])
print("grid search precision:", gs_precision)
print("grid search recall:", gs_recall)
print("grid search f-score:", gs_fscore)
print("grid search hyper:", gs_hyper)
sqlContext.clearCache()<jupyter_output>[0.1, 150]
train: 24700
fraud ratio: 0.01
train: 24600
fraud ratio: 0.01

 precision, recall, f_score: 0.5714285714285714, 0.8524590163934426, 0.6842105263157895

 precision, recall, f_score: 0.5434782608695652, 0.8064516129032258, 0.6493506493506493
train: 24600
fraud ratio: 0.01
train: 24600
fraud ratio: 0.01

 precision, recall, f_score: 0.6235294117647059, 0.8548387096774194, 0.7210884353741497

 precision, recall, f_score: 0.4049586776859504, 0.7903225806451613, 0.53551912568306
train: 24700
fraud ratio: 0.01

 precision, recall, f_score: 0.5531914893617021, 0.8524590163934426, 0.6709677419354838
[0.1, 150]
train: 4919
fraud ratio: 0.05001016466761537
train: 4939
fraud ratio: 0.05001012350678275

 precision, recall, f_score: 0.10707070707070707, 0.8548387096774194, 0.19030520646319568

 precision, recall, f_score: 0.09302325581395349, 0.9180327868852459, 0.1689291101055807
train: 4919
fraud ratio: 0.05001016466761537
train: 4919
fraud ratio: 0.05001016466761537

 precision, recall[...]<jupyter_text>The results of oversampling and undersampling are not better than no sampling method  Step9. Train base_train test base_test using hyperparams from 7)<jupyter_code>#base model after feature selection

trainData = sqlContext.read.csv("base_train.csv", header = True)
testData = sqlContext.read.csv("base_test.csv", header = True)
trainData = trainData.select(*(F.col(c).cast("double").alias(c) for c in trainData.columns))
testData = testData.select(*(F.col(c).cast("double").alias(c) for c in testData.columns))
train_count = trainData.select("Class").count()
train_fraud_count = trainData.select("Class").where(trainData.Class == 1).count()
test_count = testData.select("Class").count()
features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']
vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')
df_train = vectorAssembler.transform(trainData)
df_test = vectorAssembler.transform(testData)

lr = LogisticRegression(featuresCol = 'features', labelCol='Class', threshold=0.1, maxIter=150 )
lrModel = lr.fit(df_train)


pred = lrModel.transform(df_test)
pred = pred.select("features", "Class", "rawPrediction", "prediction")
precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()
recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()
f1 = 2 * precision * recall /(precision + recall)
print("Precision is ", precision)
print("Recall is ", recall)
print("F1 score is ", f1)<jupyter_output>Precision is  0.782608695652174
Recall is  0.7912087912087912
F1 score is  0.7868852459016393<jupyter_text>result after second grid search with final best hyper param Precision is  0.782608695652174 Recall is  0.7912087912087912 F1 score is  0.7868852459016393  Step10. train base_train test oot using hyperparams from 8)<jupyter_code>#base model after feature selection

trainData = sqlContext.read.csv("base_train.csv", header = True)
testData_oot = sqlContext.read.csv("oot.csv", header = True)
trainData = trainData.select(*(F.col(c).cast("double").alias(c) for c in trainData.columns))
testData_oot = testData_oot.select(*(F.col(c).cast("double").alias(c) for c in testData_oot.columns))
train_count = trainData.select("Class").count()
train_fraud_count = trainData.select("Class").where(trainData.Class == 1).count()
test_count_oot = testData_oot.select("Class").count()
features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']
vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')
df_train = vectorAssembler.transform(trainData)
df_test_oot = vectorAssembler.transform(testData_oot)

lr = LogisticRegression(featuresCol = 'features', labelCol='Class', threshold=0.1, maxIter=150 )
lrModel = lr.fit(df_train)


pred = lrModel.transform(df_test_oot)
pred = pred.select("features", "Class", "rawPrediction", "prediction")
precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()
recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()
f1 = 2 * precision * recall /(precision + recall)
print("Precision is ", precision)
print("Recall is ", recall)
print("F1 score is ", f1)<jupyter_output>Precision is  0.7285714285714285
Recall is  0.6891891891891891
F1 score is  0.7083333333333334
{'fraction_non_alphanumeric': 0.17542680467804814, 'fraction_numerical': 0.2204149303221759, 'mean_word_length': 5.658114558472554, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '94334', 'license': 'MIT', 'n_tokens_mistral': 11440, 'n_tokens_neox': 8378, 'n_words': 1632, 'path': 'LR_Xin.ipynb', 'repo_name': 'kvyuan/FraudTransactionDetection', 'size': '22317'}