<jupyter_start><jupyter_text>Install and Import Dependencies:<jupyter_code>!pip install kaggle

#machine learning
!pip install catboost
import pandas as pd
import numpy as np
import os
from catboost import CatBoostRegressor, Pool
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
import time

import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Import Dataset from Kaggle:<jupyter_code>from google.colab import files
uploaded = files.upload()
!mkdir -p ~/.kaggle/ && mv kaggle.json ~/.kaggle/ && chmod 600 ~/.kaggle/kaggle.json
!kaggle competitions download -c LANL-Earthquake-Prediction
!ls<jupyter_output>sample_data  sample_submission.csv  test.zip  train.csv.zip<jupyter_text>Extract Data:<jupyter_code>!unzip train.csv.zip

!unzip -d test/ test.zip<jupyter_output>Archive:  test.zip
  inflating: test/seg_430e66.csv     
  inflating: test/seg_d1a281.csv     
  inflating: test/seg_05a1b0.csv     
  inflating: test/seg_f8dd7e.csv     
  inflating: test/seg_b9bdd7.csv     
  inflating: test/seg_24c1c9.csv     
  inflating: test/seg_c5abaa.csv     
  inflating: test/seg_6262c4.csv     
  inflating: test/seg_734a88.csv     
  inflating: test/seg_94a133.csv     
  inflating: test/seg_d0c280.csv     
  inflating: test/seg_d36737.csv     
  inflating: test/seg_f80e44.csv     
  inflating: test/seg_07c815.csv     
  inflating: test/seg_7c9433.csv     
  inflating: test/seg_211486.csv     
  inflating: test/seg_78ded2.csv     
  inflating: test/seg_f11f77.csv     
  inflating: test/seg_b3883e.csv     
  inflating: test/seg_3db0a8.csv     
  inflating: test/seg_81f798.csv     
  inflating: test/seg_0a45a1.csv     
  inflating: test/seg_dc188b.csv     
  inflating: test/seg_4a9e8d.csv     
  inflating: test/seg_32fc4e.csv     
  inflating: test/seg_7b2994.cs[...]<jupyter_text>Data Analysis:<jupyter_code>train_df = pd.read_csv('train.csv', nrows = 6000000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
train_df.head()
train_ad_sample = train_df['acoustic_data'].values[::100] # :: means skip by 100
train_ttf_sample = train_df['time_to_failure'].values[::100]

# twinx() - share same X-axis but with diff scales

fig, ax1 = plt.subplots(figsize=(12, 8))

plt.plot(train_ad_sample, color='r')
ax1.set_ylabel('acoustic data', color='r')
plt.legend(['acoustic data'], loc=(0.01, 0.95))

ax2 = ax1.twinx()

plt.plot(train_ttf_sample, color='b')
ax2.set_ylabel('time to failure', color='b')
plt.legend(['time to failure'], loc=(0.01, 0.9))

plt.grid(True)<jupyter_output><empty_output><jupyter_text>Feature Engineering:<jupyter_code>def gen_features(X):
    strain = []
    #strain.append(X.mean())
    strain.append(X.std())
    strain.append(X.min())
    strain.append(X.max())
    #strain.append(X.kurtosis())
    #strain.append(X.skew())
    #strain.append(np.quantile(X,0.01))
    #strain.append(np.quantile(X,0.05))
    #strain.append(np.quantile(X,0.95))
    #strain.append(np.quantile(X,0.99))
    #strain.append(np.abs(X).min())
    strain.append(np.abs(X).max())
    #strain.append(np.abs(X).mean())
    strain.append(np.abs(X).std())
    return pd.Series(strain)<jupyter_output><empty_output><jupyter_text>Prepare Training Data:<jupyter_code>train = pd.read_csv('train.csv', iterator = True, chunksize = 150000,
                    dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float64})  #Returns TextFileReader object for Iteration
X_train = pd.DataFrame()
y_train = pd.Series()


for df in train:
  ch = gen_features(df['acoustic_data'])
  X_train = X_train.append(ch, ignore_index=True)
  y_train = y_train.append(pd.Series(df['time_to_failure'].values[-1]))
X_train.describe()
y_train.describe()
# scaling
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X_train) # compute the mean and standard deviation to be used for later scaling
X_train_scaled = scaler.transform(X_train)<jupyter_output><empty_output><jupyter_text>Training Time:<jupyter_code>#random forest

rand_forest = RandomForestRegressor(n_estimators=100, criterion='mae', oob_score=True, n_jobs=-1)
start_time = time.time()
rand_forest.fit(X_train_scaled, y_train.values.flatten())
print("Training Time:", time.time()-start_time)
print('Training Score:', rand_forest.score(X_train_scaled, y_train.values.flatten()))
rand_forest.oob_score_
rand_forest.feature_importances_
#svm

svr = SVR(kernel = 'rbf', tol = 0.01, C= 2, gamma=0.02)
start_time = time.time()
svr.fit(X_train_scaled, y_train.values.flatten())
print("Training Time:", time.time()-start_time)
print('Training Score:', svr.score(X_train_scaled, y_train.values.flatten()))
#catboost

pool = Pool(X_train, y_train)
cat_boost = CatBoostRegressor(loss_function='MAE', boosting_type='Ordered') # iterations=no. of trees
start_time = time.time()
cat_boost.fit(pool, silent=True) # don't show training | verbose
print("Time taken to train:", time.time()-start_time)
cat_boost.best_score_<jupyter_output>Time taken to train: 11.08417272567749<jupyter_text>Using LSTM:<jupyter_code>train_tfread = pd.read_csv('train.csv', iterator = True, chunksize = 150000, nrows = 6000000,
                       dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float32})
X_train_seq = []
y_train_seq = []
for df in train_tfread:
  X_train_seq.append(df['acoustic_data'].to_numpy().reshape(-1,1))
  y_train_seq.append(df['time_to_failure'].values[-1])
X_train_seq = np.array(X_train_seq)
y_train_seq = np.array(y_train_seq)
X_train_seq.shape[1:]
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, BatchNormalization
rnn = Sequential()

rnn.add(LSTM(2, input_shape=X_train_seq.shape[1:], return_sequences=True))
rnn.add(Dropout(rate=0.2))
rnn.add(BatchNormalization())

#rnn.add(LSTM(150, input_shape=X_train_seq.shape[1:], return_sequences=True))
#rnn.add(Dropout(rate=0.2))
#rnn.add(BatchNormalization())

rnn.add(LSTM(2, input_shape=X_train_seq.shape[1:]))
rnn.add(Dropout(rate=0.2))
rnn.add(BatchNormalization())

rnn.add(Dense(1, activation='elu'))
rnn.add(Dropout(rate=0.2))

rnn.add(Dense(1))

opt = tf.keras.optimizers.Adam(lr=0.001, decay=1e-6)
rnn.compile(loss='mean_absolute_error', optimizer=opt, metrics=['mae'])
rnn.summary()
time1 = time.time()
rnn.fit(X_train_seq, y_train_seq, epochs=2, batch_size=4)
print("Training Time:", time.time()-time1)
X_test_seq = []

for file in os.listdir('test'):
  data = pd.read_csv(os.path.join('test', file), dtype = {'acoustic_data': np.int16})
  X_test_seq.append(data['acoustic_data'].to_numpy().reshape(-1,1))
# This takes more than 45 min
#X_pred_rnn = rnn.predict(np.array(X_test_seq))
pd.Series(X_pred_rnn[0]).describe()<jupyter_output><empty_output><jupyter_text>**RNN didn't work out**  Best Model: Among all the models, R`andomForest` and `SVR` models were the best.Lets evaluate them on test data.Training LSTM network using sequence of 150000 also didn't work out. It was very time consuming to train and even the MAE was large.Although Rnadomforest and SVR are trained using best possible parameter values, you can fine-tune further. Or you can try other algorithms you know which can do better on this kind of dataset.  Evaluate Models:  Process test data:<jupyter_code>X_test = pd.DataFrame()
seg_ids = pd.Series()

for file in os.listdir('test'):
  data = pd.read_csv(os.path.join('test', file), dtype = {'acoustic_data': np.int16})
  data_fet = gen_features(data['acoustic_data'])
  X_test = X_test.append(data_fet, ignore_index=True)
  seg_ids = seg_ids.append(pd.Series(os.path.splitext(file)[0]))
X_test_scaled = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Make Predictions:<jupyter_code>forest_pred = rand_forest.predict(X_test_scaled)
svr_pred = svr.predict(X_test_scaled)
svr_pred
forest_pred<jupyter_output><empty_output><jupyter_text>Generate Submission File:<jupyter_code>sub = pd.DataFrame(columns=['seg_id', 'time_to_failure'])
sub['seg_id'] = seg_ids
sub['time_to_failure'] = forest_pred
sub.head()
sub.to_csv('submission_forest_latest.csv', index=False)
!ls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13642996108949415, 'fraction_numerical': 0.0316147859922179, 'mean_word_length': 5.665316045380875, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '569276', 'license': 'MIT', 'n_tokens_mistral': 3313, 'n_tokens_neox': 3116, 'n_words': 546, 'path': 'Earthquake_Prediction.ipynb', 'repo_name': 'xanjay/Earthquake-Prediction', 'size': '8224'}