<jupyter_start><jupyter_text>Imports<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates
from dateutil.relativedelta import relativedelta
import datetime<jupyter_output><empty_output><jupyter_text>Merging Data<jupyter_code>df_num_data = pd.read_csv('data/process_1/num_data.csv')
print(df_num_data.shape)
sum = df_num_data.shape[0]

for i in range(1, 6):
    read_data = pd.read_csv('data/process_' + str(i+1) + '/num_data.csv')
    print(read_data.shape)
    sum += read_data.shape[0]
    df_num_data = df_num_data.append(read_data)
    
print("sum:", sum)
print("df:", df_num_data.shape[0])
df_dot_data = pd.read_csv('data/process_1/dot_data.csv')
print(df_dot_data.shape)
sum = df_dot_data.shape[0]

for i in range(1, 6):
    read_data = pd.read_csv('data/process_' + str(i+1) + '/dot_data.csv')
    print(read_data.shape)
    sum += read_data.shape[0]
    df_dot_data = df_dot_data.append(read_data)
    
print("sum:", sum)
print("df:", df_dot_data.shape[0])
df_loss = pd.read_csv('data/process_1/loss.csv')
print(df_loss.shape)
sum = df_loss.shape[0]

for i in range(1, 6):
    read_data = pd.read_csv('data/process_' + str(i+1) + '/loss.csv')
    print(read_data.shape)
    sum += read_data.shape[0]
    df_loss = df_loss.append(read_data)
                            
print("sum:", sum)
print("df:", df_loss.shape[0])
df_outlier = pd.read_csv('data/process_1/outlier.csv')
print(df_outlier .shape)
sum = df_outlier .shape[0]

for i in range(1, 6):
    read_data = pd.read_csv('data/process_' + str(i+1) + '/outlier.csv')
    print(read_data.shape)
    sum += read_data.shape[0]
    df_outlier = df_outlier.append(read_data)
                            
print("sum:", sum)
print("df:", df_outlier.shape[0])
# total number of ionograms processed
df_num_data.shape[0] + df_dot_data.shape[0] + df_loss.shape[0] + df_outlier.shape[0]
df_num_data.head()<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code># day_1 (should be 0-3)
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = df_num_data['day_1'].hist(bins=60)
ax.set_ylabel("Ionograms")
ax.set_xlabel("day_1")

# day_1 outliers above 10
plt.subplot(1, 3, 2)
ax = df_num_data[df_num_data['day_1']>=10]['day_1'].hist(bins=50)
ax.set_xlabel("day_1")

# day_1 below 10
plt.subplot(1, 3, 3)
ax = df_num_data[df_num_data['day_1']<10]['day_1'].hist(bins=10)
ax.set_xlabel("day_1")

print(df_num_data[df_num_data['day_1']>=10].shape[0])
print(df_num_data[df_num_data['day_1']>3].shape[0])
print(df_num_data[df_num_data['day_1']<=3].shape[0])

print("% error:", 100 * df_num_data[df_num_data['day_1']>3].shape[0] / df_num_data.shape[0])
# hour_1 (should be 0-2)
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = df_num_data['hour_1'].hist(bins=60)
ax.set_ylabel("Ionograms")
ax.set_xlabel("hour_1")

# # hour_1 outliers above 10
plt.subplot(1, 3, 2)
df_num_data[df_num_data['hour_1']>=10]['hour_1'].hist(bins=50)
ax.set_xlabel("hour_1")

# # hour_1 outliers below 10
plt.subplot(1, 3, 3)
df_num_data[df_num_data['hour_1']<10]['hour_1'].hist(bins=10)
ax.set_xlabel("hour_1")

print(df_num_data[df_num_data['hour_1']>=10].shape[0])
print(df_num_data[df_num_data['hour_1']>2].shape[0])
print(df_num_data[df_num_data['hour_1']<=2].shape[0])

print("% error:", 100 * df_num_data[df_num_data['hour_1']>2].shape[0] / df_num_data.shape[0])
# minute_1 (should be 0-6)
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = df_num_data['minute_1'].hist(bins=60)
ax.set_ylabel("Ionograms")
ax.set_xlabel("minute_1")

plt.subplot(1, 3, 2)
df_num_data[df_num_data['minute_1']>6]['minute_1'].hist(bins=55)
ax.set_xlabel("minute_1")

plt.subplot(1, 3, 3)
df_num_data[df_num_data['minute_1']<=10]['minute_1'].hist(bins=10)
ax.set_xlabel("minute_1")

print(df_num_data[df_num_data['minute_1']>=10].shape[0])
print(df_num_data[df_num_data['minute_1']>6].shape[0])
print(df_num_data[df_num_data['minute_1']<=6].shape[0])

print("% error:", 100 * df_num_data[df_num_data['minute_1']>6].shape[0] / df_num_data.shape[0])
# year (should be 0-10)
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = df_num_data['year'].hist(bins=range(0,90))
ax.set_xlabel("year")
ax.set_ylabel("Ionograms")

plt.subplot(1, 3, 2)
ax = df_num_data[df_num_data['year'] < 20]['year'].hist(bins=range(0,20))
ax.set_xlabel("year")

plt.subplot(1, 3, 3)
ax = df_num_data[df_num_data['year'] <= 11]['year'].hist(bins=range(0,12))
ax.set_xlabel("year")

print(df_num_data[df_num_data['year']>10].shape[0])
print(df_num_data[df_num_data['year']<=10].shape[0])

print("% error:", 100 * df_num_data[df_num_data['year']>10].shape[0] / df_num_data.shape[0])
# station number 1 (should be 0-7)
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = df_num_data['station_number_1'].hist(bins=60)
ax.set_xlabel("station number")
ax.set_ylabel("Ionograms")

plt.subplot(1, 3, 2)
ax = df_num_data[df_num_data['station_number_1'] > 7]['station_number_1'].hist(bins=15)
ax.set_xlabel("station number")

print(df_num_data[df_num_data['station_number_1']>7].shape[0])
print(df_num_data[df_num_data['station_number_1']<=7].shape[0])

print("% error:", 100 * df_num_data[df_num_data['station_number_1']>7].shape[0] / df_num_data.shape[0])
# satellite number (should be 1)
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = df_num_data['satellite_number'].hist(bins=60)
ax.set_xlabel("satellite_number")
ax.set_ylabel("Ionograms")

plt.subplot(1, 3, 2)
ax = df_num_data[df_num_data['satellite_number'] > 1]['satellite_number'].hist(bins=50)
ax.set_xlabel("satellite_number")

plt.subplot(1, 3, 3)
ax = df_num_data[df_num_data['satellite_number'] < 10]['satellite_number'].hist(bins=10)
ax.set_xlabel("satellite_number")

print(df_num_data[df_num_data['satellite_number']>1].shape[0])
print(df_num_data[df_num_data['satellite_number']==1].shape[0])

print("% error:", 100 * df_num_data[df_num_data['satellite_number']!=1].shape[0] / df_num_data.shape[0])<jupyter_output>14487
421789
% error: 9.830987481187577<jupyter_text>Output data<jupyter_code>df_num_data.to_csv("data/all_num_data.csv")
df_dot_data.to_csv("data/all_dot_data.csv")
df_loss.to_csv("data/all_loss.csv")
df_outlier.to_csv("data/all_outlier.csv")
df_num_data = pd.read_csv("data/all_num_data.csv")<jupyter_output><empty_output><jupyter_text>Combine columns<jupyter_code>df_num_data['day'] = df_num_data.apply(lambda x: int(str(x['day_1']) + str(x['day_2']) + str(x['day_3'])), axis=1)
df_num_data['hour'] = df_num_data.apply(lambda x: int(str(x['hour_1']) + str(x['hour_2'])), axis=1)
df_num_data['minute'] = df_num_data.apply(lambda x: int(str(x['minute_1']) + str(x['minute_2'])), axis=1)
df_num_data['second'] = df_num_data.apply(lambda x: int(str(x['second_1']) + str(x['second_2'])), axis=1)
df_num_data['station_number'] = df_num_data.apply(lambda x: int(str(x['station_number_1']) + str(x['station_number_2'])), axis=1)
df_num_data.head()
rows = df_num_data.shape[0]
print("Rows in unfiltered df:", rows)
print()

filtered_df = df_num_data[df_num_data.year <= 12]
print("Errors in 'year':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.day <= 365]
print("Errors in 'day':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.hour <= 24]
print("Errors in 'hour':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.minute <= 60]
print("Errors in 'minute':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.second <= 60]
print("Errors in 'second':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.station_number <= 99]
print("Errors in 'station_number':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.satellite_number == 1]
print("Errors in 'satellite_number':", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

print()
print("Rows in filtered df:", rows)
print("Total error rate:", 100 - 100 * rows / df_num_data.shape[0])<jupyter_output>Rows in unfiltered df: 467776

Errors in 'year': 258
Errors in 'day': 32383
Errors in 'hour': 8860
Errors in 'minute': 4337
Errors in 'second': 8107
Errors in 'station_number': 194
Errors in 'satellite_number': 6332

Rows in filtered df: 407305
Total error rate: 12.92734129155835<jupyter_text>Convert to datetime object<jupyter_code>filtered_df2 = filtered_df.copy()

filtered_df['timestamp'] = filtered_df.apply(lambda x: datetime.datetime(year=1962, month=1, day=1) + \
                  relativedelta(years=x['year'], days=x['day']-1, hours=x['hour'], minutes=x['minute'], seconds=x['second']), axis=1)

filtered_df2['timestamp'] = filtered_df2.apply(lambda x: datetime.datetime(year=1960, month=1, day=1) + \
                  relativedelta(years=x['year'], days=x['day']-1, hours=x['hour'], minutes=x['minute'], seconds=x['second']), axis=1)
# 
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)

ax = filtered_df['timestamp'].hist(bins=100)

ax.set_xlabel("satellite_number")
ax.set_ylabel("Ionograms")


plt.subplot(1, 3, 2)

ax = filtered_df2['timestamp'].hist(bins=100)

ax.set_xlabel("satellite_number")
ax.set_ylabel("Ionograms")

print("Rows in df:", filtered_df.shape[0])
alouette_launch_date = datetime.datetime(year=1962, month=9, day=29)
alouette_deactivation_date = datetime.datetime(year=1972, month=12, day=31) # don't know eact date 

filtered_df = filtered_df[filtered_df.timestamp >= alouette_launch_date]
print("Errors in timestamp (date too early):", rows - filtered_df.shape[0])
rows = filtered_df.shape[0]

filtered_df = filtered_df[filtered_df.timestamp <= alouette_deactivation_date]
print("Errors in timestamp (date too late):", rows - filtered_df.shape[0])

print("Total error rate:", 100 - 100 * rows / filtered_df.shape[0])
filtered_df2 = filtered_df2[filtered_df2.timestamp >= alouette_launch_date]
print("Errors in timestamp (date too early):", rows - filtered_df2.shape[0])
rows = filtered_df2.shape[0]

filtered_df2 = filtered_df2[filtered_df2.timestamp <= alouette_deactivation_date]
print("Errors in timestamp (date too late):", rows - filtered_df2.shape[0])

print("Total error rate:", 100 - 100 * rows / filtered_df2.shape[0])
# 
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)

ax = filtered_df['timestamp'].hist(bins=100)

ax.set_xlabel("satellite_number")
ax.set_ylabel("Ionograms")


plt.subplot(1, 3, 2)

ax = filtered_df2['timestamp'].hist(bins=100)

ax.set_xlabel("satellite_number")
ax.set_ylabel("Ionograms")
# Match station_numbers with their respective station names and locations
df_stations = pd.read_csv("data/station_codes.csv")

df_stations.columns = ['station_name', 'station_number', 'before_07_01_1965','3_letter_code', 'lat', 'lon']
df_stations.astype({'station_number': 'int32'}).dtypes

stations_dict=df_stationscsvict('list')
df_stations.astype({'station_name': 'str'}).dtypes
stations_dict['station_number'].index(6)
stations_dict['station_name'][0]
#filtered_df.astype({'station_number': 'int32'}).dtypes
filtered_df.columns
type(get_station_name(1, datetime.datetime.strptime('1965-05-25 16:48:01','%Y-%m-%d %H:%M:%S')))
df = pd.DataFrame([[1, 2], [3, 4], [5, 6]], index=['a', 'b', 'c'], columns=['A', 'B'])
df['C'] = "Hello" 
type(df['C'][0])
df_stations["station_name"] = df_stations["station_name"].astype(str, errors='raise')
def get_station_name(station_number, timestamp):
    # print(station_number, timestamp)
    if timestamp >= datetime.datetime(year=1965, month=7, day=1) and station_number in stations_dict['station_number']:
        name = stations_dict['station_name'][stations_dict['station_number'].index(station_number)]
        code = stations_dict['3_letter_code'][stations_dict['station_number'].index(station_number)]
        lat = stations_dict['lat'][stations_dict['station_number'].index(station_number)]
        lon = stations_dict['lon'][stations_dict['station_number'].index(station_number)]
    elif timestamp < datetime.datetime(year=1965, month=7, day=1) and station_number in stations_dict['before_07_01_1965']:
        name = stations_dict['station_name'][stations_dict['before_07_01_1965'].index(station_number)]
        code = stations_dict['3_letter_code'][stations_dict['before_07_01_1965'].index(station_number)]
        lat = stations_dict['lat'][stations_dict['before_07_01_1965'].index(station_number)]
        lon = stations_dict['lon'][stations_dict['before_07_01_1965'].index(station_number)]
    elif timestamp < datetime.datetime(year=1963, month=4, day=25) and station_number == 10:
        name = stations_dict['station_name'][stations_dict['station_name'].index('Winkfield, England')]
        code = stations_dict['3_letter_code'][stations_dict['station_name'].index('Winkfield, England')]
        lat = stations_dict['lat'][stations_dict['station_name'].index('Winkfield, England')]
        lon = stations_dict['lon'][stations_dict['station_name'].index('Winkfield, England')]
        
    # assumption, need to look into these:
    elif timestamp >= datetime.datetime(year=1965, month=7, day=1) and station_number == 9:
        name = stations_dict['station_name'][stations_dict['station_name'].index('South Atlantic, Falkland Islands')]
        code = stations_dict['3_letter_code'][stations_dict['station_name'].index('South Atlantic, Falkland Islands')]
        lat = stations_dict['lat'][stations_dict['station_name'].index('South Atlantic, Falkland Islands')]
        lon = stations_dict['lon'][stations_dict['station_name'].index('South Atlantic, Falkland Islands')]
    elif timestamp >= datetime.datetime(year=1965, month=7, day=1) and station_number == 7:
        name = stations_dict['station_name'][stations_dict['station_name'].index('Quito, Ecuador')]
        code = stations_dict['3_letter_code'][stations_dict['station_name'].index('Quito, Ecuador')]
        lat = stations_dict['lat'][stations_dict['station_name'].index('Quito, Ecuador')]
        lon = stations_dict['lon'][stations_dict['station_name'].index('Quito, Ecuador')]
    elif timestamp >= datetime.datetime(year=1965, month=7, day=1) and station_number == 4:
        name = stations_dict['station_name'][stations_dict['station_name'].index("St. John's, Newfoundland")]
        code = stations_dict['3_letter_code'][stations_dict['station_name'].index("St. John's, Newfoundland")]
        lat = stations_dict['lat'][stations_dict['station_name'].index("St. John's, Newfoundland")]
        lon = stations_dict['lon'][stations_dict['station_name'].index("St. John's, Newfoundland")]
    else:
        name = None
        code = None
        lat = None
        lon = None
    
    #if len([name, code, lat, lon]) != 4:
        
    return name, code, lat, lon

#station_values = filtered_df.apply(lambda x: get_station_name(x['station_number'], x['timestamp']), axis=1)

filtered_df['station_name'] = None
filtered_df['3_letter_code'] = None
filtered_df['lat'] = None
filtered_df['lon'] = None

for i in range(len(filtered_df.index)):
    station_values = get_station_name(filtered_df.iloc[i]['station_number'], filtered_df.iloc[i]['timestamp'])
    
    filtered_df.iloc[i, filtered_df.columns.get_loc('station_name')] = station_values[0]
    filtered_df.iloc[i, filtered_df.columns.get_loc('3_letter_code')] = station_values[1]
    filtered_df.iloc[i, filtered_df.columns.get_loc('lat')] = station_values[2]
    filtered_df.iloc[i, filtered_df.columns.get_loc('lon')] = station_values[3]
    
#filtered_df['station_name'], filtered_df['3_letter_code'], filtered_df['lat'], filtered_df['lon'] = 
fig = plt.figure(figsize=(20, 5))

ax = filtered_df[filtered_df['station_number'] == 4].apply(lambda x: x['timestamp'].date(), axis=1).value_counts().plot()

ax.set_xlabel("station_number")
ax.set_ylabel("Ionograms")
add_value_labels(ax)
print(filtered_df.isnull().sum())
print(len(filtered_df.index))
# function from https://stackoverflow.com/questions/28931224/adding-value-labels-on-a-matplotlib-bar-chart
def add_value_labels(ax, spacing=5):
    """Add labels to the end of each bar in a bar chart.

    Arguments:
        ax (matplotlib.axes.Axes): The matplotlib object containing the axes
            of the plot to annotate.
        spacing (int): The distance between the labels and the bars.
    """

    # For each bar: Place a label
    for rect in ax.patches:
        # Get X and Y placement of label from rect.
        y_value = rect.get_height()
        x_value = rect.get_x() + rect.get_width() / 2

        # Number of points between bar and label. Change to your liking.
        space = spacing
        # Vertical alignment for positive values
        va = 'bottom'

        # If value of bar is negative: Place label below bar
        if y_value < 0:
            # Invert space to place label below
            space *= -1
            # Vertically align label at top
            va = 'top'

        # Use Y value as label and format number with one decimal place
        label = y_value

        # Create annotation
        ax.annotate(
            label,                      # Use `label` as label
            (x_value, y_value),# Place label at end of the bar
            xytext=(0, space),          # Vertically shift label by `space`
            textcoords="offset points", # Interpret `xytext` as offset in points
            ha='center',                # Horizontally center label
            va=va)                      # Vertically align label differently for
                                        # positive and negative values.
fig = plt.figure(figsize=(20, 5))

ax = filtered_df['station_name'].value_counts().plot.bar()

ax.set_xlabel("station_name")
ax.set_ylabel("Ionograms")
add_value_labels(ax)
filtered_df['station_name'].value_counts()
# St John's amd Santiago don't show up anywhere
fig = plt.figure(figsize=(20, 5))

ax = filtered_df[filtered_df['station_name'].isnull()]['station_number'].value_counts().plot.bar()

ax.set_xlabel("station_number")
ax.set_ylabel("Ionograms")
add_value_labels(ax)
#filtered_df[filtered_df['station_number'] == 7].apply(lambda x: x['timestamp'].date(), axis=1)
datetime.month(filtered_df.iloc[26338]['timestamp'].date())
print(type(list(filtered_df['station_name'])[0]))
print(type(list(df_stations['station_name'])[0]))
initial_df_size = 467776
total_error_rate = 100 * (1 - len(filtered_df.dropna().index) / initial_df_size)

print("Final df size:", len(filtered_df.dropna().index))
print("Total error rate: " + str(total_error_rate) + '%')
# fmin
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 2, 1)
ax = filtered_df['fmin'].hist(bins=100)
ax.set_xlabel("fmin")
ax.set_ylabel("Ionograms")

plt.subplot(1, 2, 2)
ax = filtered_df[filtered_df['fmin'] < 2]['fmin'].hist(bins=120)
ax.set_xlabel("fmin")
# max depth
fig = plt.figure(figsize=(20, 5))

plt.subplot(1, 3, 1)
ax = filtered_df['max_depth'].hist(bins=100, orientation='horizontal')
ax.set_ylabel("max_depth")
ax.set_xlabel("Ionograms")

plt.subplot(1, 3, 2)
ax = filtered_df[filtered_df['max_depth'] < 1600]['max_depth'].hist(bins=80, orientation='horizontal')
ax.set_ylabel("max_depth")
ax.set_xlabel("Ionograms")

plt.subplot(1, 3, 3)
ax = filtered_df[filtered_df['max_depth'] > 3000]['max_depth'].hist(bins=80, orientation='horizontal')
ax.set_ylabel("max_depth")
ax.set_xlabel("Ionograms")<jupyter_output><empty_output><jupyter_text>Fix file naming<jupyter_code>#def fix_file_name(file_name):

# dir_0 = []
# dir_1 = []
# dir_2 = []
# dir_3 = []

#     file_array = filtered_df.iloc[i]['file_name'].replace('\\', '/').split('/')
#     file_array[-3:]
df_final = filtered_df.copy()
df_final['file_name'] = filtered_df.apply(lambda x: '/'.join(x['file_name'].replace('\\', '/').split('/')[-3:])[:-4], axis=1)

#ftp://ftp.asc-csa.gc.ca/users/OpenData_DonneesOuvertes/pub/AlouetteData/Alouette%20Data/R014207815/3488-15A/1.png<jupyter_output><empty_output><jupyter_text>Drop unnecessary columns<jupyter_code>df_final.columns
df_final = df_final.drop(columns=['Unnamed: 0', 'year', 'day_1', 'day_2', 'day_3', 'hour_1','hour_2', 'minute_1', 'minute_2',\
                        'second_1', 'second_2','station_number_1', 'station_number_2', 'day', 'hour', 'minute','second'])<jupyter_output><empty_output><jupyter_text>Export final dateframe<jupyter_code>len(df_final.index)
df_final.to_csv("data/final_alouette_data.csv")<jupyter_output><empty_output><jupyter_text>Summary of results<jupyter_code>df_final_2 = pd.read_csv("data/final_alouette_data.csv")
df_final_2.isnull().sum()
len(df_final_2.index) - df_final_2["station_name"].isnull().sum()<jupyter_output><empty_output><jupyter_text>Total  of ionograms processed correctly are 394075. This is down from 504136 total ionograms with leftside metadata before processing the data. 9797 ionograms were recored as loss and an additional 26563 were recorded as outliers when going through the image processing pipeline. The reminaing ionograms were checked for their validity in this notebook via removing those that contain impossible metadata values and those with invalid station numbers. NOTE: Ionograms with metadata on the bottom are not included and were not processed as the image processing pipeline cannot yet process those.<jupyter_code>print("The overall success rate of Alouette-I ionogram processing is " + str(round(100 * 394075 / 504136, 1)) + "%")
# Pie chart
labels = 'Successfully processed ionograms', 'Image processing loss', 'Image processing outliers', 'Removed during data cleaning'
sizes = [394075, 9797, 26563, 73701]
#explode = (0, 0, 0, 0)

fig1, ax1 = plt.subplots()
ax1.pie(sizes, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90)
ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16456219293815144, 'fraction_numerical': 0.03953349556912622, 'mean_word_length': 5.85121107266436, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 69, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '38607', 'license': 'MIT', 'n_tokens_mistral': 8603, 'n_tokens_neox': 7973, 'n_words': 1374, 'path': 'data_cleaning/notebooks/alouette_data_cleaning.ipynb', 'repo_name': 'CamRoy008/AlouetteApp', 'size': '21779'}