<jupyter_start><jupyter_text><jupyter_code><jupyter_output><empty_output><jupyter_text>Downloading and unpacking Dataset<jupyter_code>!wget -O data.7z https://ti.arc.nasa.gov/c/3/
!7z e data.7z
!unrar x -Y 1st_test.rar
import glob
import datetime

import numpy as np
import pandas as pd
import scipy.stats as sts
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Preprocessing- Load dataset to pandas DataFrame- Compute basic column-wise summary statistics  - Absolute mean  - Standard deviation  - Kurtosis: Length of the tails of a signal distribution, or equivalently, how outlier prone the signal is. Developing faults can increase the number of outliers, and therefore increase the value of the kurtosis metric.  - Skewness: Asymmetry of a signal distribution. Faults can impact distribution symmetry and therefore increase the level of skewness.  - Entropy  - Root mean square<jupyter_code>DATASET_DIR_PATH = r'/content/1st_test'

COLUMNS = ['b1_x','b1_y','b2_x', 'b2_y','b3_x','b3_y','b4_x', 'b4_y']

# First 43 files were taken every 5 minutes
SKIP_FIRST_NTH_FILES = 43
SAMPLE_EVERY_NTH_ROW = 45
import numpy as np
import scipy.io

def signaltonoise(a, axis=0, ddof=0):
    a = np.asanyarray(a)
    m = a.mean(axis)
    sd = a.std(axis=axis, ddof=ddof)
    return np.where(sd == 0, 0, m/sd)

FEATURES = {
  'abs_mean': lambda x: np.abs(x).mean(),
  'std': np.std,
  'skwe': sts.skew,
  'kurt': sts.kurtosis,
  'entropy': lambda x: sts.entropy(pd.cut(x, 10).value_counts())
  ,
  'rms': lambda x: np.sqrt(np.mean(np.square(x)))
#  , 'str':signaltonoise
}

# We are taking every nth row of each file for performance reasons
def skip_row(x, n):
  if x % n == 0:
    return False
  return True

def extract_datetime_from_path(path):
  segments = path.split('/')
  filename = segments[-1]
  return datetime.datetime.strptime(filename, "%Y.%m.%d.%H.%M.%S")

def extract_col_stats_features(df, columns=COLUMNS):
  dfs_reshaped = []
  for name, func in FEATURES.items():
    applied = np.array(df.apply(func))
    columns = [c+'_'+name for c in COLUMNS]
    df_reshaped = pd.DataFrame(applied.reshape(1,8), columns=columns)
    dfs_reshaped.append(df_reshaped)

  return pd.concat(dfs_reshaped, axis=1)

def read_file(path, n):
  dfs = []
  time = 0
  files = glob.glob(DATASET_DIR_PATH + "/*")
  for filename in files:
    df_raw = pd.read_csv(filename, index_col=None, header=None, sep='\t', skiprows=lambda x:skip_row(x, n))
    df = extract_col_stats_features(df_raw)
    df['datetime'] = extract_datetime_from_path(filename)
    dfs.append(df)
  return dfs

data = pd.concat(read_file("", SAMPLE_EVERY_NTH_ROW), ignore_index=True)
data.set_index('datetime')<jupyter_output><empty_output><jupyter_text>Comparing first and last signal<jupyter_code>dataset = pd.read_csv('/content/1st_test/2003.10.22.12.06.24', sep='\t')
ax = dataset.plot(figsize = (24,6), title= "Bearing Vibration" , legend = True)
ax.set(xlabel="cycle(n)", ylabel="vibration/acceleration(g)")
plt.show()

dataset = pd.read_csv('/content/1st_test/2003.11.15.05.08.46', sep='\t')
ax = dataset.plot(figsize = (24,6), title= "Bearing Vibration" , legend = True)
ax.set(xlabel="cycle(n)", ylabel="vibration/acceleration(g)")
plt.show()<jupyter_output><empty_output><jupyter_text>Data Scoping and transforming datetime<jupyter_code>import datetime as dt

def timediff(x):
  first=dt.datetime(2003,10,22,13,59,13)
  timesince = (x - first).total_seconds()
  return  timesince/60
def timeleft(x):
  last=dt.datetime(2003,11,25,23,39,56)
  timesince = (last - x).total_seconds()
  return  abs(timesince/60)
bearing_columns = data.columns[pd.Series(data.columns).str.startswith('b4_')]
columns = bearing_columns.tolist() + ['datetime']
data = data[columns]
data.head()

data['datetime']=data['datetime'].apply(timeleft)<jupyter_output><empty_output><jupyter_text>Using SVR algorithm   - dataset dividing  - data fitting<jupyter_code>n=len(data)
data=data.sort_values(by='datetime')
data.head()
train_df_svm =  data [int(0.2*n):]
train_df_svm.head()
test_df_svm =data[0:int(n*0.2)] 

# test_df_svm['datetime']=test_df_svm['datetime'].apply(timeleft)
# train_df_svm['datetime']=train_df_svm['datetime'].apply(timeleft)
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVR
Y=train_df_svm[['datetime']].copy()
X=train_df_svm.copy()

del X['datetime']

Y_test=test_df_svm[['datetime']]
X_test=test_df_svm.copy()
del X_test['datetime']
sc_X=StandardScaler()
sc_Y=StandardScaler()
sc_X_test=StandardScaler()
# sc_Y_test=StandardScaler()




X = sc_X.fit_transform(X)

# Y = sc_Y.fit_transform(Y)
X_test=sc_X_test.fit_transform(X_test)
# Y_test=sc_Y_test.fit_transform(Y_test)<jupyter_output><empty_output><jupyter_text><jupyter_code>regressor=SVR(kernel='sigmoid')

regressor.fit(X,Y)
svm_predict=regressor.predict(X_test)
# Y_test=Y_test.to_numpy()
regressor.score(X_test,Y_test)<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)<jupyter_text>Model Scoring<jupyter_code>from sklearn.metrics import mean_squared_error, r2_score,mean_absolute_error
print(r2_score(Y_test,svm_predict),
mean_absolute_error(Y_test,svm_predict),
mean_squared_error(Y_test,svm_predict)**0.5,len(svm_predict))







bt =  sns.scatterplot(x=test_df_svm['datetime'],y=svm_predict)
bt =  sns.scatterplot(x=test_df_svm['b4_x_rms'],y=svm_predict)
bt =  sns.scatterplot(x=test_df_svm['b4_x_rms'],y=test_df_svm['datetime'])
at =  sns.scatterplot(y=data['b4_x_kurt'],x=data['datetime'])
at =  sns.scatterplot(y=data['b4_x_entropy'],x=data['datetime'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14258750857927247, 'fraction_numerical': 0.020933424845573097, 'mean_word_length': 6.25, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '398361', 'license': 'MIT', 'n_tokens_mistral': 2366, 'n_tokens_neox': 2257, 'n_words': 414, 'path': 'forecasting.ipynb', 'repo_name': 'michal-filipiak/IML-bearing-failure-forecast', 'size': '5828'}