<jupyter_start><jupyter_text>Load module<jupyter_code>import os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import matplotlib
import copy
plt.rc('text', usetex=True)<jupyter_output><empty_output><jupyter_text>big picture Attributes that we want to include+ acc:    - TIME;    - WEEKDAY; (category)    - RDSURF; (category)    - LIGHT; (category)    - WEATHER; (category)+ curv:    - deg_curv;+ grad:    - pct_grad;+ road:    - AADT;    - trkpcts;    - mvmt;    - RURURB; (category)    - MED_TYPE, MEDWID;    - LSHL_TYP, LSHL_TY2, LSHLDWID, LSHL_WD2; (2 represents decreasing direction)    - RSHL_TYP, RSHL_TY2, RSHLDWID, RSHL_WD2;    - SURF_TYP, SURF_TY2; (The composition of the driving surfacein two directions)    - lanewid, rdwy_wid; (on average, need filter > 0)    - FUNC_CLS; (? this might be an summary variable that includes info of all previous variables)(category)+ occ:    &nbsp;&nbsp;&nbsp;&nbsp;None;+ peds:    &nbsp;&nbsp;&nbsp;&nbsp;None;+ veh:    - DRV_SEX; (need to summarize)    - DRV_AGE; (need to summarize)    - vehtype; (>4 is big ones?)    - surf_typ; (Roadway surface type at the crash location? redundant info? No they do not agree)    - drv_actn; (difficult)    - intox; (need to summarize)  functions and pre-test  read function - everybody needs<jupyter_code>def detect_files(directory, keyword):
    """
    detect files in specified directory with specified keyword

    input
    -----
    directory : string
        dir to search

    keyword : string
        keyword to look for

    output
    -----
    sorted list of file names

    test
    -----
        (1) if output has larger than length;
    """

    file_list = []
    for file in os.listdir(directory):
        if not (keyword is None):
            if keyword in file:
                file_list.append(file)
        else:
            file_list.append(file)
    
    return sorted(file_list)

def read_files(directory, keyword):
    """
    read files with specified keyword

    input
    -----
    directory : string
        directory to read files from

    keyword : string
        keyword to search for

    output
    -----
    output_dic : dic
        dictionary of datasets

    test
    -----
        (1) output_dic should have length 5, for 2013 - 2017;
        (2) keyword should not be empty;
    """
    output_dic = {}
    file_list = detect_files(directory, keyword)
    for yr in range(2013, 2018):
        output_dic[yr] = pd.read_csv(os.path.join(directory, file_list[yr-2013]))
    return output_dic<jupyter_output><empty_output><jupyter_text>test on veh aggregationThis function will be used in s3_merge.py<jupyter_code>def veh_agg(df, crash_year):
    """
    aggregate vehicle info

    input
    -----
    df : pandas dataframe
        df to be summarized

    output
    -----
    df: pandas dataframe
        aggregated df

    """
    def sex(series):
        for ele in series.tolist():
            if ele > 1:
                return True
        return False

    def young(series):
        for ele in series.tolist():
            if ele < 25:
                return True
        return False

    def old(series):
        for ele in series.tolist():
            if ele > 65:
                return True
        return False

    def drink(series):
        for ele in series.tolist():
            if ele == 1.0 or ele == 5.0:
                return True
        return False

    def truck(series):
        for ele in series.tolist():
            if ele > 4:
                return True
        return False
    
    def old_car(series):
        for ele in series.tolist():
            model_year = 1900
            if ele < 10:
                model_year += (100 + ele)
            elif ele < 20:
                model_year += (100 + ele)
            else:
                model_year += (ele)
            
            if crash_year - model_year >= 15:
                return True
        return False
    
    df = df.groupby(['CASENO']).agg({'DRV_SEX': [sex],
                            'DRV_AGE': [young, old],
                            'vehtype': [truck],
                            'vehyr': [old_car],
                            # 'surf_typ': ,
                            # 'drv_actn': ,
                            'intox': [drink]
                           })
    df.columns = df.columns.get_level_values(1)
    df = df.reset_index()
    return df
veh_agg(veh[2017], 2017)<jupyter_output><empty_output><jupyter_text>analysis  read, extract, combine<jupyter_code>crash = read_files("./merged", '20')
columns = [
    'REPORT', 'ACCTYPE',
    'TIME', 'WEEKDAY', 'RDSURF', 'LIGHT', 'weather',
    'deg_curv',
    'pct_grad',
    'AADT', 'trkpcts', 'mvmt', 'RURURB', 'MED_TYPE', 'MEDWID', 
    'LSHL_TYP', 'LSHL_TY2', 'LSHLDWID', 'LSHL_WD2', 'RSHL_TYP', 'RSHL_TY2', 'RSHLDWID', 'RSHL_WD2',
    'SURF_TYP', 'SURF_TY2', 'lanewid', 'rdwy_wid', 'FUNC_CLS',
    'sex', 'young', 'old', 'drink', 'truck', 'old_car'
]

for year in crash:
    df = crash[year]
    crash[year] = df[columns]
crash[2017]
pd.unique(crash[2017]['weather'])<jupyter_output><empty_output><jupyter_text>focus on type 33***Strikes Appurtenance***Need to drop NA values. + Notice, **AADT**, **trkpcts**, **mvmt** have NA values. This is strongly undesirable;+ We drop rows with NA in those rows and thus retrieve 4,658 rows from 4,694 rows. That's not much loss;<jupyter_code>df = crash[2017]
df = df[df.ACCTYPE == 33]
# df = df.dropna()
df
df.isna().any()
df.dropna(subset=['AADT', 'trkpcts', 'mvmt'])<jupyter_output><empty_output><jupyter_text>append and obtain the final large dataset<jupyter_code>df = crash[2013]
df = df[df.ACCTYPE == 33]

for year in range(2014, 2018):
    tmp = crash[year]
    tmp.weather = tmp.weather.replace({'.': '10'})
    df = df.append(tmp[tmp.ACCTYPE == 33])

print("Before dropping, has {} rows.".format(df.shape[0]))
df = df.dropna(subset=['AADT', 'trkpcts', 'mvmt'])
print("After dropping, has {} rows.".format(df.shape[0]))<jupyter_output><empty_output><jupyter_text>write out<jupyter_code>df.to_csv('./merged/final.csv', index=False)<jupyter_output><empty_output><jupyter_text>Clearly we can see now the missing are in the types, of mdium, left shoulder, right shoulder.Interestingly, + *MEDWID* is non-zero but *MED_TYPE* is missing;+  and  are missing for many cases;+  also have a lot of missings; Maybe we remove the type attributes?We just keep the width info and don't care about the types of materials.  further delete type attributes<jupyter_code>df = pd.read_csv('./merged/final.csv')
df.isna().any()
df.isna().sum()
df = df.drop(columns=['MED_TYPE', 'LSHL_TYP', 'LSHL_TY2', 'RSHL_TYP', 'RSHL_TY2', 'SURF_TY2'])
df = df.dropna()
df.to_csv('./merged/final_no_na.csv', index=False)<jupyter_output><empty_output><jupyter_text>convert numerical to categorical and then create dummy variables The variables to be converted are:+ WEEKDAY;+ RDSURF;+ LIGHT;+ weather;+ RURURB;+ SURF_TYP;+ FUNC_CLS;<jupyter_code>df = pd.read_csv('./merged/final_no_na.csv')<jupyter_output><empty_output><jupyter_text>look at their unique values**Weather** needs some special attention.<jupyter_code>pd.unique(df.WEEKDAY)
pd.unique(df.RDSURF)
pd.unique(df.LIGHT)
pd.unique(df.weather)
pd.unique(df.RURURB)
pd.unique(df.SURF_TYP)
pd.unique(df.FUNC_CLS)<jupyter_output><empty_output><jupyter_text>convert some to integers<jupyter_code>df = df.astype(
    {'WEEKDAY':'int64', 'RDSURF':'int64', 'LIGHT':'int64', 'weather':'int64', 'FUNC_CLS':'int64'})
df['peak-hour'] = df['TIME'].apply(lambda x: 1 if (700 <= x <= 1000) or (1700 <= x <= 2000) else 0)
df['WEEKDAY'] = df['WEEKDAY'].apply(lambda x: 1 if x < 6 else 0)
df = df[df['LIGHT'].isin([1,2,3,4,5,6])]
df['LIGHT'] = df['LIGHT'].replace({5:4, 6:4})
df = df[df['AADT'] > 0]
df = df.drop(columns=['TIME'])
df = df.astype(
    {'WEEKDAY':'category', 'RDSURF':'category', 'LIGHT':'category', 
     'weather':'category', 'RURURB':'category', 'SURF_TYP':'category',
     'FUNC_CLS':'category', 'sex':'category', 'young':'category',
     'old':'category', 'drink':'category', 'truck':'category',
     'old_car':'category', 'peak-hour':'category'
    })
df.to_csv('./merged/final_type_correct.csv', index=False)<jupyter_output><empty_output><jupyter_text>SMOTE<jupyter_code>from imblearn.over_sampling import SMOTENC
from sklearn.utils import resample
from collections import Counter
# Separate majority and minority classes
crash_1 = df[df.REPORT==1]
crash_23 = df[df.REPORT!=1]

# Downsample majority class
crash_1_downsampled = resample(crash_1, 
                                 replace=False,    # sample without replacement
                                 n_samples=4843,     # to match minority class
                                 random_state=123) # reproducible results
 
# Combine minority class with downsampled majority class
crash_d = pd.concat([crash_1_downsampled, crash_23])
crash_d.columns
reg_data = crash_d[['WEEKDAY','RDSURF','LIGHT','weather','RURURB','SURF_TYP',
                    'FUNC_CLS','sex','young','old','drink','truck','old_car', 'peak-hour',
                    'MEDWID','LSHLDWID','LSHL_WD2','RSHLDWID','RSHL_WD2','lanewid','rdwy_wid',
                    'deg_curv', 'pct_grad', 'AADT', 'trkpcts', 'mvmt']]
y = crash_d['REPORT']
sm = SMOTENC(random_state=42, categorical_features=[0,1,2,3,4,5,6,7,8,9,10,11,12,13])
reg_data_res, y_res = sm.fit_resample(reg_data, y)
C = Counter(y_res)
print(C)
print(reg_data_res.shape)
print(y_res.shape)
reg_data_x = pd.DataFrame(data=reg_data_res)
reg_data_y = pd.DataFrame(data=y_res)
print(reg_data_x.shape)
print(reg_data_y)
reg_data = pd.concat([reg_data_x, reg_data_y], axis=1, sort=False)
reg_data.to_csv('./merged/final_smote.csv', index = False)<jupyter_output><empty_output><jupyter_text>create dummy and regression<jupyter_code>df = pd.read_csv('./merged/final_type_correct.csv')
df
df_log = pd.get_dummies(df, columns=['WEEKDAY', 'RDSURF', 'LIGHT', 'weather', 'RURURB', 'SURF_TYP', 'FUNC_CLS'])<jupyter_output><empty_output><jupyter_text>learningI think the sklearn logistic regression can handle this by specifying the **class_weight** as **balanced**.+ [sklarn logistic regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html)+ [a blog](https://towardsdatascience.com/machine-learning-multiclass-classification-with-imbalanced-data-set-29f6a177c1a)+ [blog code](https://github.com/javaidnabi31/Multi-class-with-imbalanced-dataset-classification/blob/master/20-news-group-classification.ipynb)+ [get dummies](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.get_dummies.html)<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LogisticRegressionCV
y = df_log['REPORT']
X = df_log.drop(columns=['REPORT', 'ACCTYPE'])
msk = np.random.rand(len(df)) < 0.8
X_train = X[msk]
X_test = X[~msk]
y_train = y[msk]
y_test = y[~msk]<jupyter_output><empty_output><jupyter_text>multinomial logistic<jupyter_code>clf = LogisticRegression(multi_class='multinomial',
                         class_weight='balanced',solver='newton-cg',
                         penalty='none'
                        ).fit(X_train, y_train)
clf.score(X_train, y_train)
prediction = clf.predict(X_test)<jupyter_output><empty_output><jupyter_text>confusion matrix<jupyter_code>from sklearn.metrics import confusion_matrix
import itertools
cnf_matrix = confusion_matrix(y_test, prediction)

fig = plt.figure(figsize=(6,6))
# fig.set_size_inches(14, 12, forward=True)
# fig.align_labels()
# fig.subplots_adjust(left=0.0, right=1.0, bottom=0.0, top=1.0)

cm = cnf_matrix
normalize = True
classes = ['PDO','INJ','FAT']

if normalize:
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    print("Normalized confusion matrix")
else:
    print('Confusion matrix, without normalization')

plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)

plt.xlim(-0.5, 2.5)
plt.ylim(-0.5, 2.5)

plt.xticks([0,1,2], classes, fontsize=15)
plt.yticks([0,1,2], classes, fontsize=15)

fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
    plt.text(j, i, format(cm[i, j], fmt),
             horizontalalignment="center",
             color="white" if cm[i, j] > thresh else "black",
             fontsize=20
            )

# plt.tight_layout()
plt.ylabel('True label', fontsize=18)
plt.xlabel('Predicted label', fontsize=18)
plt.title('Confusion matrix of multinomial logistic modelling', fontsize=22)

plt.show()<jupyter_output><empty_output><jupyter_text>multinomial logistic cv<jupyter_code>clf_cv = LogisticRegressionCV(cv=5,multi_class='multinomial', random_state=0,
                              class_weight='balanced',solver='newton-cg',
                              max_iter=200
                        ).fit(X, y)
clf_cv.score(X, y)<jupyter_output><empty_output><jupyter_text>roc curve[When to use ROC: balance](https://machinelearningmastery.com/roc-curves-and-precision-recall-curves-for-classification-in-python/)<jupyter_code>pd.unique(y_train)
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier

y_train_bin = label_binarize(y_train, classes=[1, 2, 3])
y_test_bin = label_binarize(y_test, classes=[1, 2, 3])<jupyter_output><empty_output><jupyter_text>[sklearn logistic](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html)<jupyter_code>n_classes = 3

# Learn to predict each class against the other
classifier = OneVsRestClassifier(
    LogisticRegression(
        class_weight='balanced',solver='liblinear',penalty='l2'))
y_score = classifier.fit(X_train, y_train_bin).decision_function(X_test)
from sklearn.metrics import roc_curve, auc
from scipy import interp
from itertools import cycle

# Compute ROC curve and ROC area for each class
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test_bin[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])

# Compute micro-average ROC curve and ROC area
fpr["micro"], tpr["micro"], _ = roc_curve(y_test_bin.ravel(), y_score.ravel())
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
lw = 4
# First aggregate all false positive rates
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

# Then interpolate all ROC curves at this points
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])

# Finally average it and compute AUC
mean_tpr /= n_classes

fpr["macro"] = all_fpr
tpr["macro"] = mean_tpr
roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

# Plot all ROC curves
plt.figure(figsize=(14,10))
plt.plot(fpr["micro"], tpr["micro"],
         label='micro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["micro"]),
         color='deeppink', linestyle=':', linewidth=4)

plt.plot(fpr["macro"], tpr["macro"],
         label='macro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["macro"]),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=lw,
             label='ROC curve of class {0} (area = {1:0.2f})'
             ''.format(i, roc_auc[i]))

plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.xlabel('False Positive Rate', fontsize=18)
plt.ylabel('True Positive Rate', fontsize=18)
plt.title('Some extension of Receiver operating characteristic to multi-class', fontsize=22)
plt.legend(loc="lower right",fontsize=22)
plt.show()<jupyter_output><empty_output><jupyter_text>precision-recall[Desicion-recall](https://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.htmlsphx-glr-auto-examples-model-selection-plot-precision-recall-py)<jupyter_code>from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score<jupyter_output><empty_output><jupyter_text>‘newton-cg’, ‘lbfgs’, ‘liblinear’, ‘sag’, ‘saga’<jupyter_code>clf = LogisticRegression(multi_class='multinomial',solver='newton-cg',
                         penalty='none', max_iter=200
                        ).fit(X_train, y_train)
y_score_log = clf.decision_function(X_test)
y_score_log
# For each class
precision = dict()
recall = dict()
average_precision = dict()
for i in range(3):
    precision[i], recall[i], _ = precision_recall_curve(y_test_bin[:, i],
                                                        y_score_log[:, i])
    average_precision[i] = average_precision_score(y_test_bin[:, i], y_score_log[:, i])

# A "micro-average": quantifying score on all classes jointly
precision["micro"], recall["micro"], _ = precision_recall_curve(y_test_bin.ravel(),
                                                                y_score_log.ravel())
average_precision["micro"] = average_precision_score(y_test_bin, y_score_log,
                                                     average="micro")
print('Average precision score, micro-averaged over all classes: {0:0.2f}'
      .format(average_precision["micro"]))
plt.figure(figsize=(14,10))
plt.step(recall['micro'], precision['micro'], color='b', alpha=0.2,
         where='post')
plt.fill_between(recall["micro"], precision["micro"], alpha=0.2, color='b')#,
                 #**step_kwargs)

plt.xlabel(r'Recall', fontsize=18)
plt.ylabel(r'Precision', fontsize=18)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title(r'Average precision score, micro-averaged over all classes: AP={0:0.2f}'.format(average_precision["micro"]),
          fontsize=22)
plt.show()
from itertools import cycle
# setup plot details
colors = cycle(['navy', 'turquoise', 'darkorange', 'cornflowerblue', 'teal'])

plt.figure(figsize=(12,10))
f_scores = np.linspace(0.2, 0.8, num=4)
lines = []
labels = []
for f_score in f_scores:
    x = np.linspace(0.01, 1)
    y = f_score * x / (2 * x - f_score)
    l, = plt.plot(x[y >= 0], y[y >= 0], color='gray', alpha=0.2)
    plt.annotate('f1={0:0.1f}'.format(f_score), xy=(0.9, y[45] + 0.02))

lines.append(l)
labels.append('iso-f1 curves')
l, = plt.plot(recall["micro"], precision["micro"], color='gold', lw=2)
lines.append(l)
labels.append('micro-average Precision-recall (area = {0:0.2f})'
              ''.format(average_precision["micro"]))

for i, color in zip(range(n_classes), colors):
    l, = plt.plot(recall[i], precision[i], color=color, lw=2)
    lines.append(l)
    labels.append('Precision-recall for class {0} (area = {1:0.2f})'
                  ''.format(i, average_precision[i]))

fig = plt.gcf()
fig.subplots_adjust(bottom=0.1)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('Recall', fontsize=18)
plt.ylabel('Precision', fontsize=18)
plt.title('Extension of Precision-Recall curve to multi-class',fontsize=22)
plt.legend(lines, labels, loc=(0.1, -0.4), prop=dict(size=22))


plt.show()
clf = LogisticRegression(multi_class='multinomial',solver='newton-cg',
                         penalty='none', max_iter=200,class_weight='balanced'
                        ).fit(X_train, y_train)
y_score_log = clf.decision_function(X_test)
# For each class
precision = dict()
recall = dict()
average_precision = dict()
for i in range(3):
    precision[i], recall[i], _ = precision_recall_curve(y_test_bin[:, i],
                                                        y_score_log[:, i])
    average_precision[i] = average_precision_score(y_test_bin[:, i], y_score_log[:, i])

# A "micro-average": quantifying score on all classes jointly
precision["micro"], recall["micro"], _ = precision_recall_curve(y_test_bin.ravel(),
                                                                y_score_log.ravel())
average_precision["micro"] = average_precision_score(y_test_bin, y_score_log,
                                                     average="micro")
print('Average precision score, micro-averaged over all classes: {0:0.2f}'
      .format(average_precision["micro"]))
plt.figure(figsize=(14,10))
plt.step(recall['micro'], precision['micro'], color='b', alpha=0.2,
         where='post')
plt.fill_between(recall["micro"], precision["micro"], alpha=0.2, color='b')#,
                 #**step_kwargs)

plt.xlabel(r'Recall', fontsize=18)
plt.ylabel(r'Precision', fontsize=18)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title(r'Average precision score, micro-averaged over all classes: AP={0:0.2f}'.format(average_precision["micro"]),
          fontsize=22)
plt.show()
from itertools import cycle
# setup plot details
colors = cycle(['navy', 'turquoise', 'darkorange', 'cornflowerblue', 'teal'])

plt.figure(figsize=(12,10))
f_scores = np.linspace(0.2, 0.8, num=4)
lines = []
labels = []
for f_score in f_scores:
    x = np.linspace(0.01, 1)
    y = f_score * x / (2 * x - f_score)
    l, = plt.plot(x[y >= 0], y[y >= 0], color='gray', alpha=0.2)
    plt.annotate('f1={0:0.1f}'.format(f_score), xy=(0.9, y[45] + 0.02))

lines.append(l)
labels.append('iso-f1 curves')
l, = plt.plot(recall["micro"], precision["micro"], color='gold', lw=2)
lines.append(l)
labels.append('micro-average Precision-recall (area = {0:0.2f})'
              ''.format(average_precision["micro"]))

for i, color in zip(range(n_classes), colors):
    l, = plt.plot(recall[i], precision[i], color=color, lw=2)
    lines.append(l)
    labels.append('Precision-recall for class {0} (area = {1:0.2f})'
                  ''.format(i, average_precision[i]))

fig = plt.gcf()
fig.subplots_adjust(bottom=0.1)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('Recall', fontsize=18)
plt.ylabel('Precision', fontsize=18)
plt.title('Extension of Precision-Recall curve to multi-class',fontsize=22)
plt.legend(lines, labels, loc=(0.1, -0.4), prop=dict(size=22))


plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15480072463768116, 'fraction_numerical': 0.023641304347826086, 'mean_word_length': 3.6564740615773936, 'pattern_counts': {'":': 2, '<': 94, '<?xml version=': 0, '>': 97, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '663413', 'license': 'MIT', 'n_tokens_mistral': 8422, 'n_tokens_neox': 7803, 'n_words': 1587, 'path': 'analysis.ipynb', 'repo_name': 'syin3/CET521-crash-inference', 'size': '22080'}