<jupyter_start><jupyter_text><jupyter_code>import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import tensorflow as tf
import os
from google.colab import drive
drive.mount('/gdrive')
tf.keras.backend.clear_session()
tf.random.set_seed(51)
np.random.seed(51)<jupyter_output><empty_output><jupyter_text>Mounting Drive<jupyter_code>data_names = os.listdir('/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/raw-data')<jupyter_output><empty_output><jupyter_text>Helper Function built from loading the data from drive storage after the drive has been mounted on google colab<jupyter_code>import csv
path = '/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/raw-data/'
def load_datas(dir=path,data_names = data_names):
  for i, name in enumerate(data_names):
    if i > 0:
      the_file = open(path+name, 'r')
      reader = csv.reader(the_file)
      data = []
      for i, row in enumerate(reader):
          if i >= 15:
            data.append(row)
      df_new = pd.DataFrame(data[1:],columns=data[0])
      df = pd.concat([df,df_new],axis=0,ignore_index=True)
    else:
      the_file = open(path+name, 'r')
      reader = csv.reader(the_file)
      data = []
      for i, row in enumerate(reader):
          if i >= 15:
            data.append(row)
      df = pd.DataFrame(data[1:],columns=data[0])
    df = df
  return df

df = load_datas()
df<jupyter_output><empty_output><jupyter_text>Converting the date/time column to python datatime data type<jupyter_code>df['Date/Time'] = pd.to_datetime(df['Date/Time'])<jupyter_output><empty_output><jupyter_text>Sorting rows by the Date/Time column<jupyter_code>df = df.sort_values(by='Date/Time',ignore_index=True)
pd.concat([df['Date/Time'],pd.Series(df['Temp (°C)'].values,name='Temp')],axis=1)
df
df.tail()<jupyter_output><empty_output><jupyter_text>Dropping NA rows  All rows after 2018 are NAs<jupyter_code>df = df.dropna(axis=0)<jupyter_output><empty_output><jupyter_text>Data Description<jupyter_code>df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 28021 entries, 0 to 28463
Data columns (total 24 columns):
 #   Column               Non-Null Count  Dtype         
---  ------               --------------  -----         
 0   Date/Time            28021 non-null  datetime64[ns]
 1   Year                 28021 non-null  object        
 2   Month                28021 non-null  object        
 3   Day                  28021 non-null  object        
 4   Time                 28021 non-null  object        
 5   Temp (°C)            28021 non-null  object        
 6   Temp Flag            28021 non-null  object        
 7   Dew Point Temp (°C)  28021 non-null  object        
 8   Dew Point Temp Flag  28021 non-null  object        
 9   Rel Hum (%)          28021 non-null  object        
 10  Rel Hum Flag         28021 non-null  object        
 11  Wind Dir (10s deg)   28021 non-null  object        
 12  Wind Dir Flag        28021 non-null  object        
 13  Wind Spd (km/h)      28021 non-[...]<jupyter_text>All values in weather column are NA<jupyter_code>sum(df.Weather == 'NA')<jupyter_output><empty_output><jupyter_text>Weather Column dropped no info given<jupyter_code>df = df.drop('Weather',axis=1)<jupyter_output><empty_output><jupyter_text>Columns with Missing Data<jupyter_code>columns = ['Temp Flag','Dew Point Temp Flag','Rel Hum Flag','Wind Dir Flag','Wind Spd Flag','Visibility (km)','Visibility Flag','Stn Press Flag','Hmdx','Hmdx Flag','Wind Chill Flag']<jupyter_output><empty_output><jupyter_text>Missing Values Check<jupyter_code>(df[columns] == '').sum() #As is show most of this columns data are missing<jupyter_output><empty_output><jupyter_text>Dropping the columns with over 80 percent missing data values<jupyter_code>(df[columns] == '').sum()/28011 * 100 #Over 80% data are missing for all so i will be dropping all columns
df = df.drop(columns=columns,axis=1)
df = df.replace(r'^\s*$', np.nan, regex=True)<jupyter_output><empty_output><jupyter_text>Boolean Check of Data with Missing Values<jupyter_code>df.isna().any()<jupyter_output><empty_output><jupyter_text>Percentage of Missing Values by column after dropping the first batch of missing data<jupyter_code>df.isna().sum()/28011 * 100 #72 percent of Wind Chill Data is missing thats too large so i will be dropping the column<jupyter_output><empty_output><jupyter_text>Dropping Missing Columns<jupyter_code>df = df.drop(['Wind Dir (10s deg)','Wind Chill'],axis=1)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis and Data Types Adjustment<jupyter_code>df_cols = ['Year','Month','Day','Temp (°C)','Dew Point Temp (°C)','Rel Hum (%)','Wind Spd (km/h)','Stn Press (kPa)']
df[df_cols] = df[df_cols].apply(pd.to_numeric)
df.info()
# pd.get_dummies(pd.to_datetime(df['Time'],unit='ns'))
fig,ax = plt.subplots(figsize=(12,8))
ax.plot(df['Date/Time'],df['Temp (°C)'],label='Temperature')
ax.plot(df['Date/Time'],df['Dew Point Temp (°C)'],color='orange',label='Dew Point Temp')
ax.set_title('Temperature VS Dew Point Correlation Over Time')
ax.set_xlabel('Date/Time')
ax.set_ylabel('Temp/Dew Point')
ax.legend()
df = df.set_index('Date/Time')
one_day = df['Temp (°C)'].iloc[0:24]
one_day
plt.figure(figsize=(8,6))
plt.plot(one_day)
test = df['Temp (°C)'].rolling(24).mean()
test.dropna(inplace=True)
plt.figure(figsize=(8,6))
plt.plot(df['Temp (°C)'])
plt.figure(figsize=(8,6))
plt.plot(test)<jupyter_output><empty_output><jupyter_text>Categorizing the time column<jupyter_code>pd.get_dummies(df['Time'])<jupyter_output><empty_output><jupyter_text>The Year and Time Columns are dropped<jupyter_code>df_train = df.drop(['Year','Time','Temp (°C)'],axis=1) #Dropping time for test case
temp = df['Temp (°C)']
df_train = pd.concat([df_train,temp],axis=1)<jupyter_output><empty_output><jupyter_text>Data Description<jupyter_code>df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 28021 entries, 2015-01-01 00:00:00 to 2018-03-31 23:00:00
Data columns (total 7 columns):
 #   Column               Non-Null Count  Dtype  
---  ------               --------------  -----  
 0   Month                28021 non-null  int64  
 1   Day                  28021 non-null  int64  
 2   Dew Point Temp (°C)  28020 non-null  float64
 3   Rel Hum (%)          28021 non-null  int64  
 4   Wind Spd (km/h)      28021 non-null  int64  
 5   Stn Press (kPa)      28020 non-null  float64
 6   Temp (°C)            28011 non-null  float64
dtypes: float64(3), int64(4)
memory usage: 1.7 MB<jupyter_text>Filling missing column values with the mean of 6 hr lag temperature values<jupyter_code>df_train = df_train.fillna(df_train.rolling(6,min_periods=1).mean())<jupyter_output><empty_output><jupyter_text>Cross-Checking If Missing Values are still Present<jupyter_code>df_train.isna().any()<jupyter_output><empty_output><jupyter_text>Splitting Data into Train Data and Validation Data<jupyter_code>split_time = sum(df_train.index.year == 2015) + sum(df_train.index.year == 2016) + sum(df_train.index.year == 2017)
time_train = df_train.index[:split_time]
x_train = df_train.iloc[:split_time]
time_valid = df_train.index[split_time:]
x_valid = df_train.iloc[split_time:]

window_size = 14
batch_size = 120
shuffle_buffer = 42<jupyter_output><empty_output><jupyter_text>Helper Function for preparing the dataset into a form acceptable by the model<jupyter_code>def windowed_dataset(series, window_size, batch_size, shuffle_buffer):
  dataset = tf.data.Dataset.from_tensor_slices(series)
  dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
  dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))
  dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:,:-1], window[-1,-1]))
  dataset = dataset.batch(batch_size).prefetch(1)
  return dataset<jupyter_output><empty_output><jupyter_text>Dataset Preparation and Model Build<jupyter_code>tf.keras.backend.clear_session()
dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer)
dataset_test = windowed_dataset(x_valid, window_size, batch_size, shuffle_buffer)
model = tf.keras.models.Sequential([
  tf.keras.layers.Lambda(lambda x: x * 0.1,
                      input_shape=[None,6]),
  tf.keras.layers.Conv1D(filters=32, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu"),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,activation='relu',return_sequences=True,input_shape=[None])),
  tf.keras.layers.Dropout(0.3),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,activation='relu', return_sequences=True)),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32,activation='relu')),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.Dense(16,activation='relu'),
  tf.keras.layers.Dense(1),
  tf.keras.layers.Lambda(lambda x: x * 10)
])

# 4e-4
# lr_schedule = tf.keras.callbacks.LearningRateScheduler(
#     lambda epoch: 1e-10 * 10**(epoch / 20))
optimizer = tf.keras.optimizers.Adam(learning_rate=0.00034)

# [tf.keras.metrics.RootMeanSquaredError()]
# tf.keras.losses.Huber()

model.compile(loss=tf.keras.losses.Huber(), optimizer=optimizer,metrics=[tf.keras.metrics.RootMeanSquaredError()])
history = model.fit(dataset,validation_data=dataset_test,epochs=50)<jupyter_output>Epoch 1/50
216/216 [==============================] - 36s 133ms/step - loss: 7.5125 - root_mean_squared_error: 10.0033 - val_loss: 7.4582 - val_root_mean_squared_error: 10.0400
Epoch 2/50
216/216 [==============================] - 27s 127ms/step - loss: 7.6546 - root_mean_squared_error: 10.0190 - val_loss: 5.9020 - val_root_mean_squared_error: 8.6709
Epoch 3/50
216/216 [==============================] - 27s 127ms/step - loss: 6.4184 - root_mean_squared_error: 8.4175 - val_loss: 3.7039 - val_root_mean_squared_error: 5.1655
Epoch 4/50
216/216 [==============================] - 27s 125ms/step - loss: 4.4462 - root_mean_squared_error: 6.2954 - val_loss: 6.5650 - val_root_mean_squared_error: 8.4461
Epoch 5/50
216/216 [==============================] - 27s 126ms/step - loss: 4.2483 - root_mean_squared_error: 5.9643 - val_loss: 4.9931 - val_root_mean_squared_error: 6.8296
Epoch 6/50
216/216 [==============================] - 27s 127ms/step - loss: 3.7168 - root_mean_squared_error: 5.3421 - va[...]<jupyter_text>Model Evaluation for Multi-Variate Approach<jupyter_code>epochs =  range(50)
plt.figure(figsize=(12,8))
plt.title('Epochs VS Huber Loss')
plt.plot(epochs, history.history["val_loss"],color='b',label='val_loss')
plt.plot(epochs, history.history["loss"],color='r',label='train_loss')
plt.legend()
epochs =  range(50)
plt.figure(figsize=(12,8))
plt.title('Epochs VS Root Mean Squared Error')
plt.plot(epochs, history.history["val_root_mean_squared_error"],color='b',label='val_error')
plt.plot(epochs, history.history["root_mean_squared_error"],color='r',label='train_error')
plt.legend()<jupyter_output><empty_output><jupyter_text>Saving the model and weights<jupyter_code># model.save_weights('/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/conv_net_lstm_weights_epochs_50')
# model.save('/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/conv_net_lstm_model_epochs_50')<jupyter_output><empty_output><jupyter_text>Load Saved Model<jupyter_code>new_model = tf.keras.models.Sequential([
  tf.keras.layers.Lambda(lambda x: x * 0.1,
                      input_shape=[None,6]),
  tf.keras.layers.Conv1D(filters=32, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu"),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,activation='relu',return_sequences=True,input_shape=[None])),
  tf.keras.layers.Dropout(0.3),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,activation='relu', return_sequences=True)),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32,activation='relu')),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.Dense(16,activation='relu'),
  tf.keras.layers.Dense(1),
  tf.keras.layers.Lambda(lambda x: x * 10)
])

optimizer = tf.keras.optimizers.Adam(learning_rate=0.00034)

new_model.compile(loss=tf.keras.losses.Huber(), optimizer=optimizer,metrics=[tf.keras.metrics.RootMeanSquaredError()])

# Check its architecture
new_model.summary()
new_model.load_weights("/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/model.h5")
# serialize model to JSON
model_json = new_model.to_json()
with open("/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
new_model.save_weights("/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/model.h5")
print("Saved model to disk")
# load json and create model
from tensorflow.keras.models import model_from_json
json_file = open('/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/model.h5")
print("Loaded model from disk")
loaded_model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lambda (Lambda)              (None, None, 6)           0         
_________________________________________________________________
conv1d (Conv1D)              (None, None, 32)          992       
_________________________________________________________________
bidirectional (Bidirectional (None, None, 128)         49664     
_________________________________________________________________
dropout (Dropout)            (None, None, 128)         0         
_________________________________________________________________
bidirectional_1 (Bidirection (None, None, 128)         98816     
_________________________________________________________________
bidirectional_2 (Bidirection (None, 64)                41216     
________________________________________________________[...]<jupyter_text>Train Model for one Time Lag<jupyter_code>tf.keras.backend.clear_session()
dataset = windowed_dataset(x_train, 0, batch_size, shuffle_buffer)
dataset_test = windowed_dataset(x_valid, 0, batch_size, shuffle_buffer)
model = tf.keras.models.Sequential([
  tf.keras.layers.Lambda(lambda x: x * 0.1,
                      input_shape=[None,6]),
  tf.keras.layers.Conv1D(filters=32, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu"),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,activation='relu',return_sequences=True,input_shape=[None])),
  tf.keras.layers.Dropout(0.3),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,activation='relu', return_sequences=True)),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32,activation='relu')),
  tf.keras.layers.Dropout(0.1),
  tf.keras.layers.Dense(16,activation='relu'),
  tf.keras.layers.Dense(1),
  tf.keras.layers.Lambda(lambda x: x * 10)
])

# 4e-4
# lr_schedule = tf.keras.callbacks.LearningRateScheduler(
#     lambda epoch: 1e-10 * 10**(epoch / 20))
optimizer = tf.keras.optimizers.Adam(learning_rate=0.00034)

# [tf.keras.metrics.RootMeanSquaredError()]
# tf.keras.losses.Huber()

model.compile(loss=tf.keras.losses.Huber(), optimizer=optimizer,metrics=[tf.keras.metrics.RootMeanSquaredError()])
history = model.fit(dataset,validation_data=dataset_test,epochs=23)
epochs =  range(23)
plt.figure(figsize=(12,8))
plt.title('Epochs VS Huber Loss')
plt.plot(epochs, history.history["val_loss"],color='b',label='val_loss')
plt.plot(epochs, history.history["loss"],color='r',label='train_loss')
plt.legend()
# model.save('/gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/hourly_lstm_weights_epochs_23')<jupyter_output>INFO:tensorflow:Assets written to: /gdrive/MyDrive/Colab Notebooks/Tensorflow_in_practice/Mr_Kebou_Project/models/hourly_lstm_weights_epochs_23/assets<jupyter_text>Model Forecast And Prediction Section<jupyter_code>series = x_valid.values[:,:-1]
y_valid = x_valid.values[:,-1]
def model_forecast(model, series, window_size):
    ds = tf.data.Dataset.from_tensor_slices(series)
    ds = ds.window(window_size + 1, shift=1, drop_remainder=True)
    ds = ds.flat_map(lambda w: w.batch(window_size + 1))
    ds = ds.batch(32).prefetch(1)
    forecast = model.predict(ds)
    return forecast
model_forecast(model, series[0][np.newaxis,...], 0)[0][0]
rnn_forecast = model_forecast(model, series[..., np.newaxis], 0)
rnn_forecast = rnn_forecast[:,0]
fig,ax = plt.subplots(figsize=(10, 6))
ax.set_title('Model Forecast VS Actual Forecast')
ax.plot(time_valid[window_size:], y_valid[window_size:],label='Actual Forecast')
ax.plot(time_valid[window_size:], results[window_size:],label='RNN Forecast')
ax.set_xlabel('Data/Time')
ax.set_ylabel('	Temp (°C)')
ax.legend()
fig,ax = plt.subplots(figsize=(10, 6))
ax.set_title('Model Forecast VS Actual Forecast')
ax.plot(time_valid[window_size:], y_valid[window_size:],label='Actual Forecast')
ax.plot(time_valid[window_size:], rnn_forecast,label='RNN Forecast')
ax.set_xlabel('Data/Time')
ax.set_ylabel('	Temp (°C)')
ax.legend()<jupyter_output><empty_output><jupyter_text>Looking at the prediction plot of the data above the RNN(LSTM) model performs well in forecasting future unseen temperature values, The actual future temperature value plot is in Blue while the RNN(LSTM) model Forecast Predictions is in orange<jupyter_code># plt.semilogx(history.history["lr"], history.history["loss"])
# # plt.ylim(0,4)
# # plt.axis([1e-10, 1e-4, 0, 30])<jupyter_output><empty_output><jupyter_text>Trying Out the Univariate approach to see if better forecast can be generated  Using the temperature data only for model training<jupyter_code>split_time = sum(test.index.year == 2015) + sum(test.index.year == 2016) + sum(test.index.year == 2017)
time_train = test.index[:split_time]
x_train = test.iloc[:split_time]
time_valid = test.index[split_time:]
x_valid = test.iloc[split_time:]

window_size =14
batch_size = 32
shuffle_buffer = 11<jupyter_output><empty_output><jupyter_text>Helper Function for preparing the data to a form acceptable by the model together with model building and training<jupyter_code>def window_dataset(series, window_size, batch_size, shuffle_buffer):
  dataset = tf.data.Dataset.from_tensor_slices(series)
  dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
  dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))
  dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))
  dataset = dataset.batch(batch_size).prefetch(1)
  return dataset

dataset = window_dataset(x_train, window_size, batch_size, shuffle_buffer)
model = tf.keras.models.Sequential([
  tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),
                    input_shape=[None]),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32,activation='relu', return_sequences=True)),
  tf.keras.layers.Dropout(0.35),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32,activation='relu',return_sequences=True)),
  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(16,activation='relu')),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(8,activation='relu'),
  tf.keras.layers.Dense(1)
  # tf.keras.layers.Lambda(lambda x: x * 100.0)
])

# lr_schedule = tf.keras.callbacks.LearningRateScheduler(
#     lambda epoch: 1e-8 * 10**(epoch / 20))
optimizer = optimizer = tf.keras.optimizers.Adam(learning_rate=4e-4)
model.compile(loss=tf.keras.losses.Huber(),
              optimizer=optimizer,
              metrics=[tf.keras.metrics.RootMeanSquaredError()])
# Train model
history = model.fit(dataset, epochs=100) #callbacks=[lr_schedule])<jupyter_output>Epoch 1/100
801/801 [==============================] - 36s 36ms/step - loss: 2.5712 - root_mean_squared_error: 4.7406
Epoch 2/100
801/801 [==============================] - 28s 35ms/step - loss: 0.9572 - root_mean_squared_error: 2.5542
Epoch 3/100
801/801 [==============================] - 29s 36ms/step - loss: 0.7222 - root_mean_squared_error: 2.1342
Epoch 4/100
801/801 [==============================] - 29s 36ms/step - loss: 0.4761 - root_mean_squared_error: 1.3738
Epoch 5/100
801/801 [==============================] - 29s 36ms/step - loss: 0.4145 - root_mean_squared_error: 1.3091
Epoch 6/100
801/801 [==============================] - 29s 36ms/step - loss: 0.3552 - root_mean_squared_error: 1.2262
Epoch 7/100
801/801 [==============================] - 29s 36ms/step - loss: 0.3259 - root_mean_squared_error: 1.1427
Epoch 8/100
801/801 [==============================] - 29s 36ms/step - loss: 0.3106 - root_mean_squared_error: 1.1362
Epoch 9/100
801/801 [==============================] - 2[...]<jupyter_text>Model Evaluation<jupyter_code>epochs =  range(100)
plt.figure(figsize=(12,8))
plt.plot(epochs, history.history["root_mean_squared_error"],color='r')<jupyter_output><empty_output><jupyter_text>Forecast and Prediction<jupyter_code>def model_forecast(model, series, window_size):
    ds = tf.data.Dataset.from_tensor_slices(series)
    ds = ds.window(window_size + 1, shift=1, drop_remainder=True)
    ds = ds.flat_map(lambda w: w.batch(window_size + 1))
    ds = ds.batch(32).prefetch(1)
    forecast = model.predict(ds)
    return forecast
rnn_forecast = model_forecast(model, x_valid.values[..., np.newaxis], window_size)
rnn_forecast = rnn_forecast[:,0]
fig,ax = plt.subplots(figsize=(10, 6))
ax.set_title('Model Forecast VS Actual Forecast')
ax.plot(time_valid[window_size:], x_valid.values[window_size:],label='Actual Forecast')
ax.plot(time_valid[window_size:], rnn_forecast,label='RNN Forecast')
ax.set_xlabel('Data/Time')
ax.set_ylabel('	Temp (°C)')
ax.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15709956122495136, 'fraction_numerical': 0.04396797394490433, 'mean_word_length': 5.11224771910423, 'pattern_counts': {'":': 0, '<': 133, '<?xml version=': 0, '>': 136, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '34', 'dir': 'jupyter-structured-clean-dedup', 'id': '475673', 'license': 'Apache-2.0', 'n_tokens_mistral': 8581, 'n_tokens_neox': 7696, 'n_words': 1338, 'path': 'Daily_Temperature_Forecast.ipynb', 'repo_name': 'BetikuOluwatobi/Daily-Tempearature-Forecast-Using-Recurrent-Neural-Networks', 'size': '22107'}