<jupyter_start><jupyter_text><jupyter_code>! pip install kaggle
! pip install catboost
! pip install numpy<jupyter_output>Requirement already satisfied: kaggle in /usr/local/lib/python3.6/dist-packages (1.5.4)
Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.5.3)
Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle) (2019.6.16)
Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.12.0)
Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.24.3)
Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.28.1)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.21.0)
Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle) (3.0.2)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (3.0.4)
Requirement already satisfi[...]<jupyter_text>**Import libraries**<jupyter_code># for data processing and maths operation
import pandas as pd
import numpy as np

# for ML model
from catboost import CatBoostRegressor, Pool
from sklearn.svm import SVR, NuSVR

# for scaling
from sklearn.preprocessing import StandardScaler

# for hyper model tuning
from sklearn.model_selection import GridSearchCV

# for kernel grid
from sklearn.kernel_ridge import KernelRidge

# for visualisation
import matplotlib.pyplot as plt

from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error

print(pd.__version__)
print(np.__version__)<jupyter_output>0.24.2
1.16.4<jupyter_text>**Import files**<jupyter_code>from google.colab import files
uploaded = files.upload()

! mkdir -p ~/.kaggle/ && mv kaggle.json ~/.kaggle/ && chmod 600 ~/kaggle/
# kaggle competeion list
! kaggle competitions list
# Download the dataset

! kaggle competitions download -c LANL-Earthquake-Prediction
# check if data is there mow
# !mkdir test
# !unzip test.zip -d test/
# !unzip train.csv.zip
!ls<jupyter_output>sample_data  sample_submission.csv  test  test.zip  train.csv  train.csv.zip<jupyter_text>**Data Analysis**<jupyter_code>#Extract training data into a dataframe for further manipulation
train = pd.read_csv('train.csv', nrows=200000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
print(train.shape)
train.head()
rows = 15
segments = int(np.floor(train.shape[0] / rows))

X_train = pd.DataFrame(index=range(segments), dtype=np.float64,
                       columns=['ave', 'std', 'max', 'min'])
y_train = pd.DataFrame(index=range(segments), dtype=np.float64,
                       columns=['time_to_failure'])

 
for segment in tqdm(range(segments)):
    seg = train.iloc[segment*rows:segment*rows+rows]
    x = seg['acoustic_data'].values
    y = seg['time_to_failure'].values[-1]
    
    y_train.loc[segment, 'time_to_failure'] = y
    
    X_train.loc[segment, 'ave'] = x.mean()
    X_train.loc[segment, 'std'] = x.std()
    X_train.loc[segment, 'max'] = x.max()
    X_train.loc[segment, 'min'] = x.min()
X_train.shape
y_train.shape
X_train.head(10)
#visualize 1% of samples data, first 100 datapoints
train_ad_sample_df = train['acoustic_data'].values[::100]
train_ttf_sample_df = train['time_to_failure'].values[::100]

#function for plotting based on both features
def plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title="Acoustic data and time to failure: 1% sampled data"):
    fig, ax1 = plt.subplots(figsize=(12, 8))
    plt.title(title)
    plt.plot(train_ad_sample_df, color='r')
    ax1.set_ylabel('acoustic data', color='r')
    plt.legend(['acoustic data'], loc=(0.01, 0.95))
    ax2 = ax1.twinx()
    plt.plot(train_ttf_sample_df, color='b')
    ax2.set_ylabel('time to failure', color='b')
    plt.legend(['time to failure'], loc=(0.01, 0.9))
    plt.grid(True)

plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)
del train_ad_sample_df
del train_ttf_sample_df<jupyter_output><empty_output><jupyter_text>Feature **Enginnering**<jupyter_code># Step 4 - Feature Engineering and signifiance of these statistical features

#lets create a function to generate some statistical features based on the training data
def gen_features(X):
    strain = []
    strain.append(X.mean())
    strain.append(X.std())
    strain.append(X.min())
    strain.append(X.max())
    strain.append(X.kurtosis())
    strain.append(X.skew())
    strain.append(np.quantile(X,0.01))
    strain.append(np.quantile(X,0.05))
    strain.append(np.quantile(X,0.95))
    strain.append(np.quantile(X,0.99))
    strain.append(np.abs(X).max())
    strain.append(np.abs(X).mean())
    strain.append(np.abs(X).std())
    return pd.Series(strain)
train = pd.read_csv('train.csv', iterator=True, chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})

X_train = pd.DataFrame()
y_train = pd.Series()
for df in train:
    ch = gen_features(df['acoustic_data'])
    X_train = X_train.append(ch, ignore_index=True)
    y_train = y_train.append(pd.Series(df['time_to_failure'].values[-1]))
X_train.describe()
train_pool = Pool(X_train, y_train)

# for interation in list([100, 5000, 10000, 20000]):
#   m = CatBoostRegressor(iterations=interation, loss_function='MAE', boosting_type='Ordered')
#   m.fit(X_train, y_train, silent=True)
#   print('Iteration {}, {}'.format(interation, m.best_score_))

m = CatBoostRegressor(iterations=10000, loss_function='MAE', boosting_type='Ordered')
m.fit(X_train, y_train, silent=True)
print('Iteration {}, {}'.format(10000, m.best_score_))
y_pred = m.predict(X_train)
score = mean_absolute_error(y_train.values.flatten(), y_pred)
print(f'Score: {score:0.3f}')
submission = pd.read_csv('sample_submission.csv', index_col='seg_id')

X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)

for seg_id in X_test.index:
    seg = pd.read_csv('test/' + seg_id + '.csv')
    
    x = seg['acoustic_data'].values
    
    X_test.loc[seg_id, 'ave'] = x.mean()
    X_test.loc[seg_id, 'std'] = x.std()
    X_test.loc[seg_id, 'max'] = x.max()
    X_test.loc[seg_id, 'min'] = x.min()
X_test.head(10)
scaler = StandardScaler()
scaler.fit(X_train)
X_test_scaled = scaler.transform(X_test)
submission['time_to_failure'] = m.predict(X_test_scaled)
submission.to_csv('submission.csv')
m.predict(X_test_scaled)
submissionDf = pd.read_csv('submission.csv')
submissionDf.head()<jupyter_output><empty_output><jupyter_text>**Submit result to Kaggle**<jupyter_code>!kaggle competitions submit -c LANL-Earthquake-Prediction -f submission.csv -m "First submission using catboost"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14245932874102302, 'fraction_numerical': 0.027553861937564122, 'mean_word_length': 5.181159420289855, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '613554', 'license': 'MIT', 'n_tokens_mistral': 2605, 'n_tokens_neox': 2503, 'n_words': 497, 'path': 'LANL-Earthquake-Prediction/kernel.ipynb', 'repo_name': 'myke11j/kaggle-competetions', 'size': '6823'}