<jupyter_start><jupyter_text>Imports Import Libraries<jupyter_code>import tarfile
import pandas as pd
import os
# Import Libraries for section "Linear regression model"
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from math import sqrt
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).<jupyter_text>Set Your Directory<jupyter_code>YOUR_DIRECTORY = '/content/drive/My Drive/Colab Notebooks/ATMS597/project4/' #Sarah
# YOUR_DIRECTORY = '/content/drive/My Drive/Colab Notebooks/ATMS 597/P04/' #Cathy
# YOUR_DIRECTORY = '/content/drive/My Drive/ATMS597 Weather Climate Data Analysis/Module 4/Project 4/' #Chu-Chun<jupyter_output><empty_output><jupyter_text>Import GFS data - Daily<jupyter_code>daily = tarfile.open(name = YOUR_DIRECTORY + 'daily.tar.gz') # Set the archive for opening

# Aggregate to PD DataFrame
cur_file = daily.next() # Initiate while loop using the first file in the tar archive
daily_gfs = pd.DataFrame(columns=['TMAX', 'TMIN', 'WMAX', 'RTOT'])
i = 0
while cur_file != None:
    i += 1
    if i % 350 == 0:
        print(float(i/3500))
    working_file = YOUR_DIRECTORY + cur_file.name
    daily.extract(cur_file, path=YOUR_DIRECTORY) # Extract TarInfo Object
    convert_to_df = pd.read_csv(working_file, index_col=0, parse_dates=True, 
                                infer_datetime_format=True) # Convert cur_file 
                                # (TarInfo Object) to string, then to PD; convert
                                # index col to DateTime
    daily_gfs = daily_gfs.append(convert_to_df) # Append PD to DF
    os.remove(working_file) # Remove file extracted in directory
    cur_file = daily.next() # Go to next file in archive

daily.close() # Close .tar
daily_gfs
# Find missing dates
missing_daily_gfs = pd.date_range(start = '2010-01-01 12:00:00', end = '2020-01-31 12:00:00', freq='D').difference(daily_gfs.index)
missing_daily_gfs<jupyter_output><empty_output><jupyter_text>Import GFS data - Profiles<jupyter_code># Profile gfs data
prof = tarfile.open(name = YOUR_DIRECTORY + 'prof.tar.gz') # Set the archive for opening

# Aggregate to PD DataFrame
cur_file = prof.next() # Initiate while loop using the first file in the tar archive
prof_gfs = pd.DataFrame(columns=['DWPC','HGHT','PRES','TMPC','UWND','VWND'])
i = 0
while cur_file != None:
    i += 1
    if i % 350 == 0:
        print(float(i/3500))
    working_file = YOUR_DIRECTORY + cur_file.name
    prof.extract(cur_file, path=YOUR_DIRECTORY) # Extract TarInfo Object
    convert_to_df = pd.read_csv(working_file, index_col=0, parse_dates=True, 
                                infer_datetime_format=True) # Convert cur_file 
                                # (TarInfo Object) to string, then to PD; convert
                                # index col to DateTime
    prof_gfs = prof_gfs.append(convert_to_df) # Append PD to DF
    os.remove(working_file) # Remove file extracted in directory
    cur_file = prof.next() # Go to next file in archive

prof.close() # Close .tar
prof_gfs
# Find missing times
missing_prof_gfs = pd.date_range(start = '2010-01-02 06:00:00', end = '2020-02-02 06:00:00', freq='6H').difference(prof_gfs.index)
missing_prof_gfs

# the result shows that len(missing_prof_gfs) < 4*len(missing_daily_gfs)...not sure how to go from here.<jupyter_output><empty_output><jupyter_text>Import GFS data - Surface<jupyter_code># Surface gfs data
sfc = tarfile.open(name = YOUR_DIRECTORY + 'sfc.tar.gz') # Set the archive for opening

# Aggregate to PD DataFrame
cur_file = sfc.next() # Initiate while loop using the first file in the tar archive
sfc_gfs = pd.DataFrame()
i = 0
while cur_file != None:
    i += 1
    if i % 350 == 0:
        print(float(i/3500))
    working_file = YOUR_DIRECTORY + cur_file.name
    sfc.extract(cur_file, path=YOUR_DIRECTORY) # Extract TarInfo Object
    convert_to_df = pd.read_csv(working_file, index_col=0).T # Convert cur_file 
                                # (TarInfo Object) to string, then to PD. Note that 
                                # the sfc files are transposed, i.e. they have 
                                # variables as rows and timestamps as columns, hence .T
    sfc_gfs = sfc_gfs.append(convert_to_df) # Append PD to DF
    os.remove(working_file) # Remove file extracted in directory
    cur_file = sfc.next() # Go to next file in archive

sfc.close() # Close .tar
sfc_gfs
# Find missing times
missing_sfc_gfs = pd.date_range(start = '2010-01-02 06:00:00', end = '2020-02-02 06:00:00', freq='3H').difference(sfc_gfs.index)
missing_sfc_gfs

# the result shows that len(missing_sfc_gfs) < (24/3)*len(missing_daily_gfs)...not sure how to go from here.<jupyter_output><empty_output><jupyter_text>Import obs daily data<jupyter_code>daily_obs = pd.read_csv(YOUR_DIRECTORY + 'KCMI_daily.csv', header=4, usecols=[0,1,2,3,4], index_col='Date')[:-7]  # ignore the last 7 lines
daily_obs
# Check for missing dates - there's none
missing_dates_obs = pd.date_range(start = '2010-01-01', end = '2019-12-31', freq='D').difference(daily_obs.index)
missing_dates_obs<jupyter_output><empty_output><jupyter_text>Import obs hourly data<jupyter_code>hourly_obs = pd.read_csv(YOUR_DIRECTORY + 'KCMI_hourly.csv', #header=1, usecols=[0,1,2,3,4], comment='#', 
                         index_col=0, parse_dates=True, infer_datetime_format=True)
hourly_obs<jupyter_output><empty_output><jupyter_text>Resample hourly precip data into daily freq and add to daily_obs<jupyter_code>hourly_obs[hourly_obs['pr1h'] < 0] = 0 # Counts trace precip (-0.1) as 0
hourly_obs_res = hourly_obs.resample('24H',base=6).sum()
precip_daily = hourly_obs_res['pr1h']['2010-01-01 06:00:00':'2019-12-31 06:00:00'].resample('D').sum()
precip_daily
daily_obs['Total Precip from Hourly (in)'] = precip_daily
daily_obs<jupyter_output><empty_output><jupyter_text>TMAX  Plot the TMAX from GFS and observation<jupyter_code>GFS_TMAX = daily_gfs['TMAX']['2010-01-01 12:00:00':'2018-12-30 12:00:00']  # select 2010-01-01 to 2018-12-30
GFS_TMAX.index = GFS_TMAX.index.strftime('%Y-%m-%d')  # to be consistent with observation index
print(GFS_TMAX)
GFS_TMAX.plot()
# Select 2010-01-02 to 2018-12-31 (one day after GFS model) and Mask Missing Data
mask_missing = daily_obs['Max Hourly Temp (F)'] == 'M'
OBS_TMAX = daily_obs['Max Hourly Temp (F)'].mask(mask_missing).dropna().astype(float)['2010-01-02':'2018-12-31']
print(OBS_TMAX)
OBS_TMAX.plot()<jupyter_output>Date
2010-01-02     6.0
2010-01-03    10.0
2010-01-04    11.0
2010-01-05    10.0
2010-01-06    18.0
              ... 
2018-12-26    45.0
2018-12-27    54.0
2018-12-28    51.0
2018-12-29    28.0
2018-12-30    36.0
Name: Max Hourly Temp (F), Length: 3261, dtype: float64<jupyter_text>Delete missing days for GFS daily, GFS sfc, and Obs Find the overlap dates between GFS daily and observation (one day after GFS model) and delete the missing dates.<jupyter_code>GFS_TMAX_plus1day = pd.to_datetime(GFS_TMAX.index) + pd.Timedelta('1 day')  # OBS dates are one day after GFS timestamps
mismatch_dates = GFS_TMAX_plus1day.difference(pd.to_datetime(OBS_TMAX.index))  # OBS doesn't have these dates
OBS_TMAX_dates = GFS_TMAX_plus1day.drop(mismatch_dates)  # dates derived from available GFS dates and OBS dates
GFS_TMAX_dates = OBS_TMAX_dates - pd.Timedelta('1 day')<jupyter_output><empty_output><jupyter_text>Update the dates in both GFS and obs TMAX<jupyter_code>GFS_TMAX[GFS_TMAX_dates.strftime('%Y-%m-%d')]
OBS_TMAX[OBS_TMAX_dates.strftime('%Y-%m-%d')]<jupyter_output><empty_output><jupyter_text>Find the max temperature from sfc GFS dataset between 06Z to 06Z<jupyter_code>sfc_gfs.index = pd.to_datetime(sfc_gfs.index, format='%Y-%m-%d %H:%M:%S')
GFS_sfc_TMPC_daily_max = sfc_gfs['TMPC'].astype(float).resample('24H',base=6).max()['2010-01-02 06:00:00':'2018-12-31 06:00:00'].resample('D').mean()
GFS_sfc_TMPC_daily_max<jupyter_output><empty_output><jupyter_text>Drop the dates that don't exist<jupyter_code>mismatch_dates = pd.to_datetime(GFS_sfc_TMPC_daily_max.index).difference(pd.to_datetime(OBS_TMAX_dates))  # OBS_TMAX_dates doesn't have these dates
GFS_sfc_TMPC_max = GFS_sfc_TMPC_daily_max.drop(mismatch_dates)
GFS_sfc_TMPC_max<jupyter_output><empty_output><jupyter_text>TMIN  Plot the TMIN from GFS and observation<jupyter_code>GFS_TMIN = daily_gfs['TMIN']['2010-01-01 12:00:00':'2018-12-30 12:00:00']  # select 2010-01-01 to 2018-12-30
GFS_TMIN.index = GFS_TMIN.index.strftime('%Y-%m-%d')  # to be consistent with observation index
print(GFS_TMIN)
GFS_TMIN.plot()
# Select 2010-01-02 to 2018-12-31 (one day after GFS model) and Mask Missing Data
mask_missing = daily_obs['Min Hourly Temp (F)'] == 'M'
OBS_TMIN = daily_obs['Min Hourly Temp (F)'].mask(mask_missing).dropna().astype(float)['2010-01-02':'2018-12-31']
print(OBS_TMIN)
OBS_TMIN.plot()<jupyter_output>Date
2010-01-02    -5.0
2010-01-03    -6.0
2010-01-04    -2.0
2010-01-05    -1.0
2010-01-06    -3.0
              ... 
2018-12-26    24.0
2018-12-27    39.0
2018-12-28    28.0
2018-12-29    23.0
2018-12-30    18.0
Name: Min Hourly Temp (F), Length: 3261, dtype: float64<jupyter_text>Delete missing days for GFS daily, GFS sfc, and Obs Find the overlap dates between GFS daily and observation (one day after GFS model) and delete the missing dates.<jupyter_code>GFS_TMIN_plus1day = pd.to_datetime(GFS_TMIN.index) + pd.Timedelta('1 day')  # OBS dates are one day after GFS timestamps
mismatch_dates = GFS_TMIN_plus1day.difference(pd.to_datetime(OBS_TMIN.index))  # OBS doesn't have these dates
OBS_TMIN_dates = GFS_TMIN_plus1day.drop(mismatch_dates)  # dates derived from available GFS dates and OBS dates
GFS_TMIN_dates = OBS_TMIN_dates - pd.Timedelta('1 day')<jupyter_output><empty_output><jupyter_text>Update the dates in both GFS and obs TMAX<jupyter_code>GFS_TMIN[GFS_TMIN_dates.strftime('%Y-%m-%d')]
OBS_TMIN[OBS_TMIN_dates.strftime('%Y-%m-%d')]<jupyter_output><empty_output><jupyter_text>Find the min temperature from sfc GFS dataset between 06Z to 06Z<jupyter_code>sfc_gfs.index = pd.to_datetime(sfc_gfs.index, format='%Y-%m-%d %H:%M:%S')
GFS_sfc_TMPC_daily_min = sfc_gfs['TMPC'].astype(float).resample('24H',base=6).min()['2010-01-02 06:00:00':'2018-12-31 06:00:00'].resample('D').mean()
GFS_sfc_TMPC_daily_min<jupyter_output><empty_output><jupyter_text>Drop the dates that don't exist<jupyter_code>mismatch_dates = pd.to_datetime(GFS_sfc_TMPC_daily_min.index).difference(pd.to_datetime(OBS_TMIN_dates))  # OBS_TMAX_dates doesn't have these dates
GFS_sfc_TMPC_min = GFS_sfc_TMPC_daily_min.drop(mismatch_dates)
GFS_sfc_TMPC_min<jupyter_output><empty_output><jupyter_text>Max Wind  Plot the WMAX from GFS and observation<jupyter_code>GFS_WMAX = daily_gfs['WMAX']['2010-01-01 12:00:00':'2018-12-30 12:00:00']  # select 2010-01-01 to 2018-12-30
GFS_WMAX.index = GFS_WMAX.index.strftime('%Y-%m-%d')  # to be consistent with observation index
print(GFS_WMAX)
GFS_WMAX.plot()
# Select 2010-01-02 to 2018-12-31 (one day after GFS model) and Mask Missing Data
mask_missing = daily_obs['Max Wind Speed (mph)'] == 'M'
OBS_WMAX = daily_obs['Max Wind Speed (mph)'].mask(mask_missing).dropna().astype(float)['2010-01-02':'2018-12-31']
print(OBS_WMAX)
OBS_WMAX.plot()<jupyter_output>Date
2010-01-02    15.0
2010-01-03    11.0
2010-01-04    20.0
2010-01-05    16.0
2010-01-06     8.0
              ... 
2018-12-26    15.0
2018-12-27    33.0
2018-12-28    23.0
2018-12-29    16.0
2018-12-30    21.0
Name: Max Wind Speed (mph), Length: 3264, dtype: float64<jupyter_text>Delete missing days for GFS daily, GFS sfc, and Obs Find the overlap dates between GFS daily and observation (one day after GFS model) and delete the missing dates.<jupyter_code>GFS_WMAX_plus1day = pd.to_datetime(GFS_WMAX.index) + pd.Timedelta('1 day')  # OBS dates are one day after GFS timestamps
mismatch_dates = GFS_WMAX_plus1day.difference(pd.to_datetime(OBS_WMAX.index))  # OBS doesn't have these dates
OBS_WMAX_dates = GFS_WMAX_plus1day.drop(mismatch_dates)  # dates derived from available GFS dates and OBS dates
GFS_WMAX_dates = OBS_WMAX_dates - pd.Timedelta('1 day')<jupyter_output><empty_output><jupyter_text>Update the dates in both GFS and obs TMAX<jupyter_code>GFS_WMAX[GFS_WMAX_dates.strftime('%Y-%m-%d')]
OBS_WMAX[OBS_WMAX_dates.strftime('%Y-%m-%d')]<jupyter_output><empty_output><jupyter_text>Find the max wind from sfc GFS dataset between 06Z to 06Z<jupyter_code>sfc_gfs.index = pd.to_datetime(sfc_gfs.index, format='%Y-%m-%d %H:%M:%S')
GFS_sfc_WMAX_daily = sfc_gfs['WSPD'].astype(float).resample('24H',base=6).max()['2010-01-02 06:00:00':'2018-12-31 06:00:00'].resample('D').mean()
GFS_sfc_WMAX_daily<jupyter_output><empty_output><jupyter_text>Drop the dates that don't exist<jupyter_code>mismatch_dates = pd.to_datetime(GFS_sfc_WMAX_daily.index).difference(pd.to_datetime(OBS_WMAX_dates))  # OBS_TMAX_dates doesn't have these dates
GFS_sfc_WMAX = GFS_sfc_WMAX_daily.drop(mismatch_dates)
GFS_sfc_WMAX<jupyter_output><empty_output><jupyter_text>Precipitation  Plot the WMAX from GFS and observation<jupyter_code>daily_gfs[daily_gfs['RTOT'] < 0] = 0 # Takes care of trace precip
GFS_PRCP = daily_gfs['RTOT']['2010-01-01 12:00:00':'2018-12-30 12:00:00']  # select 2010-01-01 to 2018-12-30
GFS_PRCP.index = GFS_PRCP.index.strftime('%Y-%m-%d')  # to be consistent with observation index
print(GFS_PRCP)
GFS_PRCP.plot()
# Select 2010-01-02 to 2018-12-31 (one day after GFS model) and Mask Missing Data
mask_missing = daily_obs['Total Precip from Hourly (in)'] == 'M'
OBS_PRCP = daily_obs['Total Precip from Hourly (in)'].mask(mask_missing).dropna().astype(float)['2010-01-02':'2018-12-31']
print(OBS_PRCP)
OBS_PRCP.plot()<jupyter_output>Date
2010-01-02     0.0
2010-01-03     0.0
2010-01-04     0.0
2010-01-05     0.0
2010-01-06     5.6
              ... 
2018-12-27    13.9
2018-12-28     0.0
2018-12-29     0.0
2018-12-30     9.0
2018-12-31    27.4
Name: Total Precip from Hourly (in), Length: 3286, dtype: float64<jupyter_text>Delete missing days for GFS daily, GFS sfc, and Obs Find the overlap dates between GFS daily and observation (one day after GFS model) and delete the missing dates.<jupyter_code>GFS_PRCP_plus1day = pd.to_datetime(GFS_PRCP.index) + pd.Timedelta('1 day')  # OBS dates are one day after GFS timestamps
mismatch_dates = GFS_PRCP_plus1day.difference(pd.to_datetime(OBS_PRCP.index))  # OBS doesn't have these dates
OBS_PRCP_dates = GFS_PRCP_plus1day.drop(mismatch_dates)  # dates derived from available GFS dates and OBS dates
GFS_PRCP_dates = OBS_PRCP_dates - pd.Timedelta('1 day')<jupyter_output><empty_output><jupyter_text>Update the dates in both GFS and obs TMAX<jupyter_code>GFS_PRCP[GFS_PRCP_dates.strftime('%Y-%m-%d')]
OBS_PRCP[OBS_PRCP_dates.strftime('%Y-%m-%d')]<jupyter_output><empty_output><jupyter_text>Find the total precip from sfc GFS dataset between 06Z to 06Z<jupyter_code>sfc_gfs.index = pd.to_datetime(sfc_gfs.index, format='%Y-%m-%d %H:%M:%S')
sfc_gfs[sfc_gfs['PRCP'] < 0] = 0 # Takes care of trace precip
GFS_sfc_PRCP_daily = sfc_gfs['PRCP'].astype(float).resample('24H',base=6).sum()['2010-01-02 06:00:00':'2018-12-31 06:00:00'].resample('D').mean()
GFS_sfc_PRCP_daily<jupyter_output><empty_output><jupyter_text>Drop the dates that don't exist<jupyter_code>mismatch_dates = pd.to_datetime(GFS_sfc_PRCP_daily.index).difference(pd.to_datetime(OBS_PRCP_dates))  # OBS_TMAX_dates doesn't have these dates
GFS_sfc_PRCP = GFS_sfc_PRCP_daily.drop(mismatch_dates)
GFS_sfc_PRCP<jupyter_output><empty_output><jupyter_text>Linear regression model  TMAX<jupyter_code>X = np.column_stack((GFS_TMAX[GFS_TMAX_dates.strftime('%Y-%m-%d')].values, GFS_sfc_TMPC_max.values))  # GFS model daily and sfc max TMPC
y = (OBS_TMAX[OBS_TMAX_dates.strftime('%Y-%m-%d')].values-32)*5/9  # Observation, converted from F to C
model = LinearRegression(fit_intercept=False)
model.fit(X, y)
y_predict = model.predict(X)  # linear regression model prediction

print("Model slope:    ", model.coef_)
print("Model intercept:", model.intercept_)
# y_predict = X[:, 0] * model.coef_[0] + X[:, 1] * model.coef_[1] + model.intercept_  # another form to write the equation

plt.figure(figsize=(20, 8.5))
# plt.plot(OBS_TMAX_dates, X, alpha=0.5)
plt.plot(OBS_TMAX_dates, y, alpha=0.5, label='Observation')
plt.plot(OBS_TMAX_dates, y_predict, alpha=0.5, label='Prediction')
plt.legend()
plt.show()

rms = sqrt(mean_squared_error(y, y_predict))
print('RMS: ', rms)<jupyter_output>Model slope:     [0.81910761 0.25291388]
Model intercept: 0.0<jupyter_text>TMIN<jupyter_code>X = np.column_stack((GFS_TMIN[GFS_TMIN_dates.strftime('%Y-%m-%d')].values, GFS_sfc_TMPC_min.values))  # GFS model daily and sfc min TMPC
y = (OBS_TMIN[OBS_TMIN_dates.strftime('%Y-%m-%d')].values-32)*5/9  # Observation, converted from F to C

model = LinearRegression(fit_intercept=False)
model.fit(X, y)
y_predict = model.predict(X)  # linear regression model prediction

print("Model slope:    ", model.coef_)
print("Model intercept:", model.intercept_)
# y_predict = X[:, 0] * model.coef_[0] + X[:, 1] * model.coef_[1] + model.intercept_  # another form to write the equation

plt.figure(figsize=(20, 8.5))
# plt.plot(OBS_TMIN_dates, X, alpha=0.5)
plt.plot(OBS_TMIN_dates, y, alpha=0.5, label='Observation')
plt.plot(OBS_TMIN_dates, y_predict, alpha=0.5, label='Prediction')
plt.legend()
plt.show()

rms = sqrt(mean_squared_error(y, y_predict))
print('RMS: ', rms)<jupyter_output>Model slope:     [0.87602057 0.13003458]
Model intercept: 0.0<jupyter_text>Max Wind<jupyter_code>X = np.column_stack((GFS_WMAX[GFS_WMAX_dates.strftime('%Y-%m-%d')].values, GFS_sfc_WMAX.values))  # GFS model daily and sfc WMAX
y = (OBS_WMAX[OBS_WMAX_dates.strftime('%Y-%m-%d')].values)*0.44704  # Observation, converted from mph to m/s

model = LinearRegression(fit_intercept=False)
model.fit(X, y)
y_predict = model.predict(X)  # linear regression model prediction

print("Model slope:    ", model.coef_)
print("Model intercept:", model.intercept_)
# y_predict = X[:, 0] * model.coef_[0] + X[:, 1] * model.coef_[1] + model.intercept_  # another form to write the equation

plt.figure(figsize=(20, 8.5))
# plt.plot(OBS_WMAX_dates, X, alpha=0.5)
plt.plot(OBS_WMAX_dates, y, alpha=0.5, label='Observation')
plt.plot(OBS_WMAX_dates, y_predict, alpha=0.5, label='Prediction')
plt.legend()
plt.show()

rms = sqrt(mean_squared_error(y, y_predict))
print('RMS: ', rms)<jupyter_output>Model slope:     [0.89309483 0.39048531]
Model intercept: 0.0<jupyter_text>Precipitation<jupyter_code>X = np.column_stack((GFS_PRCP[GFS_PRCP_dates.strftime('%Y-%m-%d')].values, GFS_sfc_PRCP.values))  # GFS model daily and sfc total PRCP
y = (OBS_PRCP[OBS_PRCP_dates.strftime('%Y-%m-%d')].values)*0.039370  # Observation, converted from mm to inches

model = LinearRegression(fit_intercept=False)
model.fit(X, y)
y_predict = model.predict(X)  # linear regression model prediction

print("Model slope:    ", model.coef_)
print("Model intercept:", model.intercept_)
# y_predict = X[:, 0] * model.coef_[0] + X[:, 1] * model.coef_[1] + model.intercept_  # another form to write the equation

plt.figure(figsize=(20, 8.5))
# plt.plot(OBS_PRCP_dates, X, alpha=0.5)
plt.plot(OBS_PRCP_dates, y, alpha=0.5, label='Observation')
plt.plot(OBS_PRCP_dates, y_predict, alpha=0.5, label='Prediction')
plt.legend()
plt.show()

rms = sqrt(mean_squared_error(y, y_predict))
print('RMS: ', rms)<jupyter_output>Model slope:     [ 0.03007177 -0.01040771]
Model intercept: 0.0
{'fraction_non_alphanumeric': 0.1321689392778714, 'fraction_numerical': 0.06536949083295031, 'mean_word_length': 5.081366459627329, 'pattern_counts': {'":': 0, '<': 125, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '32', 'dir': 'jupyter-structured-clean-dedup', 'id': '493102', 'license': 'MIT', 'n_tokens_mistral': 8564, 'n_tokens_neox': 7626, 'n_words': 1554, 'path': 'Developing_ipynb/project4_0327_SS.ipynb', 'repo_name': 'chuchun2/ATMS-597-Project-4-Group-C', 'size': '19581'}