<jupyter_start><jupyter_text>Generate column RUL (Remaining Useful Life)<jupyter_code>f = lambda x: 2 if x <= w0 else (1 if x <= w1 else 0)
f(31)
# Window for different target classes
w1 = 30
w0 = 15

def assign_rul(group):
    return group.max() - group

def assign_label1(val):
    if val <= w1:
        return 1
    else:
        return 0
    
def assign_label2(val):
    if val <= w0:
        return 2
    elif val <= w1:
        return 1
    else:
        return 0

df['RUL'] = df.groupby(level='id')['cycle'].transform(assign_rul)
df['RUL'] = df.groupby(level='id').transform(assign_rul)
df['label1'] = df['RUL'].apply(assign_label1)
df['label2'] = df['RUL'].apply(assign_label2)
df.head()
filter_sensor_cols = [x for x in df.columns if re.search('s\d', x) is not None]
sensor_data = df[filter_sensor_cols]
df_rollmean = sensor_data.groupby(level='id', as_index=False).apply(lambda grp: grp.rolling(5, min_periods=1).mean())
df_rollmean.columns = ['{}_rollmean'.format(x) for x in df_rollmean.columns]
df_rollstd = sensor_data.groupby(level='id', as_index=False).apply(lambda grp:grp.rolling(5, min_periods=1).std()).fillna(0)
df_rollstd.columns = ['{}_rollstd'.format(x) for x in df_rollstd.columns]
df = pd.concat([df, df_rollmean, df_rollstd], axis=1)
# Normalize with min_max
def min_max(col):
    minval = min(col)
    maxval = max(col)
    rangeval = maxval - minval
    return (col - minval) / rangeval

cols_for_normalization = [x for x in df.columns if x not in ['id', 'label1', 'label2']]
df = df[cols_for_normalization].apply(min_max).fillna(0).dropna(how='all')
df[:5]
min_max_scaler = preprocessing.MinMaxScaler()
min_max_scaler.fit(df[cols_for_normalization])
df[cols_for_normalization] = min_max_scaler.transform(df[cols_for_normalization])
# Select features based on Pearson correlation
from scipy.stats import pearsonr
features = df[[x for x in cols_for_normalization if x != 'RUL']]
features['s2'].corr(df['RUL'])
feature_cols = [x[0] for x in sorted(corr, key=lambda x: abs(x[1]),reverse=True)[:35]]
train_X, train_y = df[feature_cols].values, df['RUL'].values
train_X.shape, train_y.shape
from sklearn.ensemble import RandomForestRegressor
estimator = RandomForestRegressor(n_estimators=8, max_depth=32)
estimator.fit(train_X, train_y)
from __future__ import print_function
for x in sorted(corr, key=lambda tupl: abs(tupl[1]), reverse=True):
    print(x[0])
import numpy as np
corr = []
for col in features.columns:
    corrval = (col, features[col].corr(df['RUL']))
    if not np.isnan(corrval[1]):
        corr.append(corrval)
df['corr'] = features.corrwith(df['RUL'])
df[:5]
features['cycle'].corr(features['s2'])
pearsonr([1,2,5], [3,4,3])
text = "RUL	cycle	a4	a11	a21	a15	a20	a17	a12	a7	a2	a3	s11	s4	s12	s7	s15	s21	s20	s2	s17	a8	a13	s3	s8	s13	a9	s9	a14	s14	sd6	a6	s6	sd14	sd9	sd13"
cols = []
for c in text.split():
    if c.startswith('a'):
        c = 's' + c[1:] + '_rollmean'
    elif c.startswith('sd'):
        c = 's' + c[2:] + '_rollstd'
    cols.append(c)

for c in cols:
    print(c)
truth = pd.read_table('PM_truth.txt', sep='\s+', names=['RUL'])
truth[:3]
def add_labels(df, w0=15, w1=30):
    df['label1'] = df['RUL'].apply(lambda x: 1 if x <= w1 else 0)
    df['label2'] = df['RUL'].apply(lambda x: 2 if x <= w0 else (1 if x <= w1 else 0))
    return df
add_labels(truth)
def add_rul(df):
    # Remaining useful life
    df['RUL'] = df.groupby('id')['cycle'].transform(lambda grp: grp.max() - grp)
    return df


def add_labels(df, w0=15, w1=30):
    df['label1'] = df['RUL'].apply(lambda x: 1 if x <= w1 else 0)
    df['label2'] = df['RUL'].apply(lambda x: 2 if x <= w0 else (1 if x <= w1 else 0))
    return df


def add_rolling_values(agg_func, result_prefix, df, window_size=5):
    # Sensor data all in s1,s2,s3... pattern
    sensor_colnames = [x for x in df.columns if x == 'id' or re.search('s\d', x) is not None]
    sensor_df = df[sensor_colnames]
    new_df = sensor_df.groupby('id').transform(agg_func).fillna(0)
    # Column names for rolling mean will be a1,a2,a3...
    new_df.columns = [result_prefix + x[1:] for x in new_df.columns]
    df = pd.concat([df, new_df], axis=1)
    return df


def select_top_features(df, measure='pearson_correlation', n=35):
    if measure != 'pearson_correlation':
        raise ValueError('Only pearson correlation is supported for now.')
    excluded_colnames = ['id', 'label1', 'label2', 'RUL']
    feature_colnames = [x for x in df.columns if x not in excluded_colnames]
    feature_corr = []
    for col in feature_colnames:
        corr = df[col].corr(df['RUL'])
        if not np.isnan(corr):
            feature_corr.append((col, corr))
    top_corr = sorted(feature_corr, key=lambda x: abs(x[1]), reverse=True)[:n]
    return [colname for colname, val in top_corr]
window_size = 5
df_test = pd.read_table('PM_test.txt', sep='\s+', names=colnames)
df_truth = pd.read_table('PM_truth.txt', sep='\s+', names=['RUL'])
df_test = add_rolling_values(lambda grp: grp.rolling(window_size, min_periods=1).mean(), 'a', df_test)
df_test = add_rolling_values(lambda grp: grp.rolling(window_size, min_periods=1).std(), 'sd', df_test)
df_test = df_test[df_test.groupby('id')['cycle'].transform(max) == df_test['cycle']]
df_test.index = range(len(df_test))
df_test
# Append RUL, label1, label2 to test data
df_truth = add_labels(df_truth)
pd.concat([df_test,df_truth], axis=1)
pd.concat([df_test, df_truth], axis=1)
df_truth<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15695644183054586, 'fraction_numerical': 0.030325307847822093, 'mean_word_length': 4.513677811550152, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '640349', 'license': 'MIT', 'n_tokens_mistral': 2208, 'n_tokens_neox': 2117, 'n_words': 514, 'path': 'Preprocessing.ipynb', 'repo_name': 'davidheryanto/azureml-predictive-maintenance', 'size': '5441'}