<jupyter_start><jupyter_text>DatasetInput - timestamp, values, exogenous variablesOutput - start, end anomalous intervals, severity  Loading the dataset<jupyter_code>X_train = load_signal('multivariate/C-1-train')
X_test = load_signal('multivariate/C-1-test')

X_train.shape, X_test.shape
X_test
1284832800 + 21600
X_train.timestamp.iloc[-1]
AnomalyDataset.load('NASA', 'MSL', 'C-1').test.data
with open('processed/SWaT.pickle', 'rb') as f:
    dataset = pickle.load(f)
    
interval = 25 * 60
split = len(dataset['X']) // 3
X_train = dataset['X'][:split]
X_train['timestamp'] = X_train.index
y_train = dataset['y'][:split]<jupyter_output>/Users/lcwong/opt/anaconda3/envs/orion-env/lib/python3.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>Preprocessing<jupyter_code># Creates an equi-spaced time series by aggregating values over fixed specified interval
# Food for thought, we don't need to bin it by set intervals?
params = {
    "time_column": "timestamp", 
    "interval": 21600, 
    # "interval": interval,
    "method": "mean"
}
primitive = load_primitive('mlprimitives.custom.timeseries_preprocessing.time_segments_aggregate', arguments=params)
X, index = primitive.produce(X=X_train)

X.shape, index.shape
# This primitive is an imputation transformer for filling missing values
params = {
    'X': X
}
primitive = load_primitive('sklearn.impute.SimpleImputer', arguments=params)
primitive.fit()
X = primitive.produce(X=X)

X.shape
# This primitive transforms features by scaling each feature to a given range
params = {
    "feature_range": [-1, 1], 
    'X': X,
}
primitive = load_primitive('sklearn.preprocessing.MinMaxScaler', arguments=params)
primitive.fit()
X = primitive.produce(X=X)

X.shape
# Uses a rolling window approach to create the sub-sequences out of time series data
params = {
    "target_column": 0, 
    "window_size": 100, 
    'target_size': 1, 
    'step_size': 1
}
primitive = load_primitive('mlprimitives.custom.timeseries_preprocessing.rolling_window_sequences',
                           arguments=params)
X, y, index, target_index = primitive.produce(X=X, index=index)


# Target / target size is the next interval that is trying to predict.
# Index is the start of the interval
X.shape, y.shape, index.shape, target_index.shape
# Target
params = {
    "target_index": 0, 
    "axis": 2
}
primitive = load_primitive('orion.primitives.timeseries_preprocessing.slice_array_by_dims',
                           arguments=params)
y = primitive.produce(X=X)

# Trying to predict the target sequence which is the first column of X
X.shape, y.shape<jupyter_output><empty_output><jupyter_text>Saving the Data<jupyter_code>np.save('processed/data/X', X)
np.save('processed/data/y', y)
np.save('processed/data/index', index)<jupyter_output><empty_output><jupyter_text>NASA<jupyter_code>df = pd.read_csv('datasets/NASA/source/labeled_anomalies.csv')
df.head()
MSL_dataset = {
    'train': [],
    'test': [],
    'anomaly': [],
}

SMAP_dataset = {
    'train': [],
    'test': [],
    'anomaly': [],
}

for split in ['train', 'test']:

    for filepath in glob.glob(f'datasets/NASA/source/{split}/**.npy'):
        filename = os.path.basename(filepath)
        signal, _ = filename.split('.')

        info = df[df.chan_id == signal]
    
        if len(info) > 0:
            dataset = pd.DataFrame(np.load(filepath))
            dataset = dataset.reset_index() 
            
            # Metadata
            original_columns = dataset.columns
            dataset['source'] = 'NASA'
            dataset['name'] = info['spacecraft'].iloc[0]
            dataset['signal'] = signal
            
            # Anomaly points
            anomaly_points = [0] * len(dataset)
            if split == 'test':
                for interval in eval(info.anomaly_sequences.iloc[0]):
                    for i in range(interval[0], interval[1]):
                        if i < len(anomaly_points):
                            anomaly_points[i] = 1
            dataset['anomaly'] = anomaly_points
            dataset = dataset[list(dataset.columns[-4:]) + list(dataset.columns[:-4])]
            
            # Anomaly intervals
            anomaly_interval = pd.DataFrame(eval(info.anomaly_sequences.iloc[0]))
            anomaly_interval.columns = ['start', 'end']
            anomaly_interval['source'] = 'NASA'
            anomaly_interval['name'] = name
            anomaly_interval['signal'] = signal
            anomaly_interval = anomaly_interval[list(anomaly_interval.columns[-3:]) + list(anomaly_interval.columns[:-3])]

            if info['spacecraft'].iloc[0] == 'SMAP':
                SMAP_dataset[split].append(dataset)
                if split == 'test':
                    SMAP_dataset['anomaly'].append(anomaly_interval)
            elif info['spacecraft'].iloc[0] == 'MSL':
                MSL_dataset[split].append(dataset)
                if split == 'test':
                    MSL_dataset['anomaly'].append(anomaly_interval)

SMAP_dataset['train'] = pd.concat(SMAP_dataset['train'])
SMAP_dataset['test'] = pd.concat(SMAP_dataset['test'])
SMAP_dataset['anomaly'] = pd.concat(SMAP_dataset['anomaly']).drop_duplicates()
MSL_dataset['train'] = pd.concat(MSL_dataset['train'])
MSL_dataset['test'] = pd.concat(MSL_dataset['test'])
MSL_dataset['anomaly'] = pd.concat(MSL_dataset['anomaly']).drop_duplicates()
def print_summary(train, test, anomaly):
    print('# channels', len(train.columns[5:]))
    print('# contextual', len(anomaly))
    print('# Anomaly', train.anomaly.value_counts().get(1, 0) + test.anomaly.value_counts().get(1, 0))
    print('# Data', len(train) + len(test))
SMAP_dataset['train'].head()
SMAP_dataset['anomaly'].head()
print_summary(SMAP_dataset['train'], SMAP_dataset['test'], SMAP_dataset['anomaly'])
print_summary(MSL_dataset['train'], MSL_dataset['test'], MSL_dataset['anomaly'])
with open('processed/datasets/MSL.pickle', 'wb') as f:
    pickle.dump(MSL_dataset, f)
with open('processed/datasets/SMAP.pickle', 'wb') as f:
    pickle.dump(SMAP_dataset, f)<jupyter_output><empty_output><jupyter_text>SWaT<jupyter_code>df = pd.read_excel('datasets/SWaT/A1_A2_2015_dec/SWaT_Dataset_Attack_v0.xlsx')
df.columns = df.iloc[0]
df = df.iloc[1:]
df = df.set_index(pd.to_datetime(df[' Timestamp'], dayfirst=True))
df.head()
raw_abnormalities = """
28/12/2015 10:29:14	10:44:53
28/12/2015 10:51:08	10:58:30 
28/12/2015 11:22:00	11:28:22
28/12/2015 11:47:39	11:54:08
28/12/2015 12:00:55	12:04:10
28/12/2015 12:08:25	12:15:33
28/12/2015 13:10:10	13:26:13
28/12/2015 14:16:20	14:19:00
28/12/2015 14:19:00	14:28:20
29/12/2015 11:11:25	11:15:17
29/12/2015 11:35:40	11:42:50
29/12/2015 11:57:25	12:02:00
29/12/2015 14:38:12	14:50:08
29/12/2015 18:10:43	18:15:01
29/12/2015 18:15:43	18:22:17
29/12/2015 18:30:00	18:42:00
29/12/2015 22:55:18	23:03:00
30/12/2015 01:42:34	1:54:10
30/12/2015 09:51:08	9:56:28
30/12/2015 10:01:50	10:12:01
30/12/2015 17:04:56	17:29:00
31/12/2015 01:17:08	1:45:18
31/12/2015 01:45:19	11:15:27
31/12/2015 15:32:00	15:34:00
31/12/2015 15:47:40	16:07:10
31/12/2015 22:05:34	22:11:40
1/01/2016 10:36:00	10:46:00
1/01/2016 14:21:12	14:28:35
1/01/2016 17:12:40	17:14:20
1/01/2016 17:18:56	17:26:56
1/01/2016 22:16:01	22:25:00
2/01/2015 11:17:02	11:24:50
2/01/2015 11:31:38	11:36:18
2/01/2015 11:43:48	11:50:28
2/01/2015 11:51:42	11:56:38
2/01/2015 13:13:02	13:40:56
"""
split = raw_abnormalities.split()
abnormalities = {
    'start': [],
    'end': []
}
for i in range(0, len(split), 3):
    date, start, end = split[i], split[i+1], split[i+2]
    abnormalities['start'].append(date + ' ' + start)
    abnormalities['end'].append(date + ' ' + end)
    
abnormalities = pd.DataFrame(abnormalities)

abnormalities['source'] = 'SUTD'
abnormalities['name'] = 'SWaT'
abnormalities['signal'] = None
abnormalities['start'] = pd.to_datetime(abnormalities['start'], dayfirst=True)
abnormalities['end'] = pd.to_datetime(abnormalities['end'], dayfirst=True)

abnormalities = abnormalities[list(abnormalities.columns[-3:]) + list(abnormalities.columns[:-3])]

abnormalities.head()
y = []
for idx, abnormality in abnormalities.iterrows():
    sub_df = pd.DataFrame(pd.date_range(abnormality['start'], abnormality['end'], freq='1s'))
    sub_df['target'] = 1
    y.append(sub_df)
y = pd.concat(y).set_index(0)
final_df = df.iloc[:, 1:52].reset_index()
final_df = final_df.rename(columns={' Timestamp': 'index'})

final_df['source'] = 'SUTD'
final_df['name'] = 'SWaT'
final_df['signal'] = None
final_df['anomaly'] = df.merge(y, left_index=True, right_index=True, how='left').drop_duplicates()['target'].fillna(0).values
final_df = final_df[list(final_df.columns[-4:]) + list(final_df.columns[:-4])]
split = len(final_df) * 2 // 3
train = final_df.iloc[:split]
test = final_df.iloc[split:]
SWAT_dataset = {
    'train': train,
    'test': test,
    'anomaly': abnormalities
}
print_summary(SWAT_dataset['train'], SWAT_dataset['test'], SWAT_dataset['anomaly'])
with open('processed/datasets/SWaT.pickle', 'wb') as f:
    pickle.dump(SWAT_dataset, f)
with open('processed/datasets/SWaT.pickle', 'rb') as f:
    SWAT_dataset_2 = pickle.load(f)<jupyter_output><empty_output><jupyter_text>WADI<jupyter_code>df = pd.read_csv('datasets/WADI/A1_2017_oct/WADI_attackdataLABLE.csv', header=1)
df = df[:-3]
hour = []
counter = -1 
for t in df.Time:
    if t == '00:00.0':
        counter += 1
    if counter % 24 < 10:
        hour.append(f'0{counter % 24}')
    else:
        hour.append(str(counter % 24))
df['hour'] = hour
datetime = (df['Date '] + ' ' + df['hour']+ ':' + df['Time']).apply(lambda x: str(x)[:-2])
df = df.set_index(pd.to_datetime(datetime, dayfirst=True))
raw_abnormalities = """
9/10/17 19:25:00 19:50:16
10/10/17 10:24:10 10:34:00
10/10/17 10:55:00 11:24:00
10/10/17 11:30:40 11:44:50
10/10/17 13:39:30 13:50:40
10/10/17 14:48:17 14:59:55
10/10/17 17:40:00 17:49:40
11/10/17 10:55:00 10:56:27
11/10/17 11:17:54 11:31:20
11/10/17 11:36:31 11:47:00
11/10/17 11:59:00 12:05:00
11/10/17 12:07:30 12:10:52
11/10/17 12:16:00 12:25:36
11/10/17 15:26:30 15:37:00
"""
split = raw_abnormalities.split()
abnormalities = {
    'start': [],
    'end': []
}
for i in range(0, len(split), 3):
    date, start, end = split[i], split[i+1], split[i+2]
    abnormalities['start'].append(date + ' ' + start)
    abnormalities['end'].append(date + ' ' + end)
    
abnormalities = pd.DataFrame(abnormalities)
abnormalities['source'] = 'SUTD'
abnormalities['name'] = 'WADI'
abnormalities['signal'] = None
abnormalities['start'] = pd.to_datetime(abnormalities['start'], dayfirst=True)
abnormalities['end'] = pd.to_datetime(abnormalities['end'], dayfirst=True)

abnormalities = abnormalities[list(abnormalities.columns[-3:]) + list(abnormalities.columns[:-3])]

abnormalities.head()
y = []
for idx, abnormality in abnormalities.iterrows():
    sub_df = pd.DataFrame(pd.date_range(abnormality['start'], abnormality['end'], freq='1s'))
    sub_df['target'] = 1
    y.append(sub_df)
y = pd.concat(y).set_index(0)
final_df = df.iloc[:, 3:130].fillna(0).reset_index()
final_df = final_df.rename(columns={' Timestamp': 'index'})

final_df['source'] = 'SUTD'
final_df['name'] = 'WADI'
final_df['signal'] = None
final_df['anomaly'] = df.merge(y, left_index=True, right_index=True, how='left').drop_duplicates()['target'].fillna(0).values
final_df = final_df[list(final_df.columns[-4:]) + list(final_df.columns[:-4])]
split = len(final_df) * 2 // 3
train = final_df.iloc[:split]
test = final_df.iloc[split:]
WADI_dataset = {
    'train': train,
    'test': test,
    'anomaly': abnormalities
}
print_summary(WADI_dataset['train'], WADI_dataset['test'], WADI_dataset['anomaly'])
with open('processed/datasets/WADI.pickle', 'wb') as f:
    pickle.dump(WADI_dataset, f)<jupyter_output><empty_output><jupyter_text>Yahoo<jupyter_code>A1_dataset = {
    'train': [],
    'test': [],
    'anomaly': [],
}

for filepath in glob.glob(f'datasets/YAHOO/A1Benchmark/**.csv'):
    df = pd.read_csv(filepath)
    df['source'] = 'Yahoo'
    df['name'] = 'A1'
    
    filename = os.path.basename(filepath)
    signal, _ = filename.split('.')
    df['signal'] = signal
    df = df[['source', 'name', 'signal', 'is_anomaly', 'timestamp', 'value']]
    df = df.rename(columns={'is_anomaly': 'anomaly', 'timestamp': 'index'})

    anomaly_index = []
    start = None
    prev = None
    for index, anomaly in zip(df['index'], df.anomaly):
        if start is None and anomaly == 1:
            start = [index]
        elif start is not None and anomaly == 0:
            start.append(prev)
            anomaly_index.append(start)
            start = None
        elif start is not None and index == len(df):
            start.append(index)
            anomaly_index.append(start)
        prev = index
        

    anomaly_index = pd.DataFrame(anomaly_index)
    if len(anomaly_index.columns == 2):
        anomaly_index.columns = ['start', 'end']
        anomaly_index['source'] = 'Yahoo'
        anomaly_index['name'] = 'A1'
        anomaly_index['signal'] = signal
        anomaly_index = anomaly_index[['source', 'name', 'signal', 'start', 'end']]
        
    split = (len(df) * 2) // 3
    train, test = df.iloc[:split], df.iloc[split:]
    A1_dataset['train'].append(train)
    A1_dataset['test'].append(test)
    A1_dataset['anomaly'].append(anomaly_index)

A1_dataset['train'] = pd.concat(A1_dataset['train'])
A1_dataset['test'] = pd.concat(A1_dataset['test'])
A1_dataset['anomaly'] = pd.concat(A1_dataset['anomaly'])
print_summary(A1_dataset['train'], A1_dataset['test'], A1_dataset['anomaly'])
b = sum(A1_dataset['anomaly']['start'] == A1_dataset['anomaly']['end'])
a = len(A1_dataset['anomaly'])
print(a, b, a-b)
with open('processed/datasets/A2.pickle', 'wb') as f:
    pickle.dump(A1_dataset, f)
A3_dataset = {
    'train': [],
    'test': [],
    'anomaly': [],
}

for filepath in glob.glob(f'datasets/YAHOO/A4Benchmark/A4Benchmark-**.csv'):
    df = pd.read_csv(filepath)

    df['source'] = 'Yahoo'
    df['name'] = 'A4'
    
    filename = os.path.basename(filepath)
    signal, _ = filename.split('.')
    df['signal'] = signal
    df = df[['source', 'name', 'signal', 'anomaly', 'timestamps', 'value', 'changepoint', 'trend', 'noise',
       'seasonality1', 'seasonality2', 'seasonality3']]
    df = df.rename(columns={'is_anomaly': 'anomaly', 'timestamps': 'index'})

    anomaly_index = []
    start = None
    prev = None
    for index, anomaly in zip(df['index'], df.anomaly):
        if start is None and anomaly == 1:
            start = [index]
        elif start is not None and anomaly == 0:
            start.append(prev)
            anomaly_index.append(start)
            start = None
        elif start is not None and index == len(df):
            start.append(index)
            anomaly_index.append(start)
        prev = index
        

    anomaly_index = pd.DataFrame(anomaly_index)
    if len(anomaly_index.columns == 2):
        anomaly_index.columns = ['start', 'end']
        anomaly_index['source'] = 'Yahoo'
        anomaly_index['name'] = 'A4'
        anomaly_index['signal'] = signal
        anomaly_index = anomaly_index[['source', 'name', 'signal', 'start', 'end']]
        
    split = (len(df) * 2) // 3
    train, test = df.iloc[:split], df.iloc[split:]
    A3_dataset['train'].append(train)
    A3_dataset['test'].append(test)
    A3_dataset['anomaly'].append(anomaly_index)

A3_dataset['train'] = pd.concat(A3_dataset['train'])
A3_dataset['test'] = pd.concat(A3_dataset['test'])
A3_dataset['anomaly'] = pd.concat(A3_dataset['anomaly'])
print_summary(A3_dataset['train'], A3_dataset['test'], A3_dataset['anomaly'])
b = sum(A3_dataset['anomaly']['start'] == A3_dataset['anomaly']['end'])
a = len(A3_dataset['anomaly'])
print(a, b, a-b)
with open('processed/datasets/A4.pickle', 'wb') as f:
    pickle.dump(A3_dataset, f)<jupyter_output><empty_output><jupyter_text>NAB<jupyter_code>with open('datasets/NAB/labels/combined_windows.json', 'rb') as f:
    labels = json.load(f)
dataset = {}
for filepath, anomalies in labels.items():
    category, filename = filepath.split('/')
    if len(anomalies) != 0:
        dataset.setdefault(category, dict())
        dataset[category].setdefault(filename, [])
        dataset[category][filename].extend(anomalies)
Artificial_dataset = {
    'train': [],
    'test': [],
    'anomaly': [],
}

for filepath in glob.glob(f'datasets/NAB/data/realTweets/**.csv'):
    df = pd.read_csv(filepath)
    df = df.rename(columns={'timestamp': 'index'})
    df['source'] = 'NAB'
    df['name'] = 'Tweets'
    
    filename = os.path.basename(filepath)
    signal, _ = filename.split('.')
    df['signal'] = signal
    
    df['anomaly'] = 0
    for interval in dataset['realTweets'].get(filename, []):
        cond = (interval[0] <= df['index']) & (df['index'] <= interval[1])
        df.loc[cond, 'anomaly'] = df.loc[cond, 'anomaly'] + 1
    
    df = df[['source', 'name', 'signal', 'anomaly', 'index', 'value']]
    
    
    anomaly_index = pd.DataFrame(dataset['realTweets'].get(filename, []))
    if len(anomaly_index.columns == 2):
        anomaly_index.columns = ['start', 'end']
        anomaly_index['source'] = 'NAB'
        anomaly_index['name'] = 'Tweets'
        anomaly_index['signal'] = signal
        anomaly_index = anomaly_index[['source', 'name', 'signal', 'start', 'end']]
        
    
    
    split = (len(df) * 2) // 3
    train, test = df.iloc[:split], df.iloc[split:]
    Artificial_dataset['train'].append(train)
    Artificial_dataset['test'].append(test)
    Artificial_dataset['anomaly'].append(anomaly_index)

    
Artificial_dataset['train'] = pd.concat(Artificial_dataset['train'])
Artificial_dataset['test'] = pd.concat(Artificial_dataset['test'])
Artificial_dataset['anomaly'] = pd.concat(Artificial_dataset['anomaly'])
print_summary(Artificial_dataset['train'], Artificial_dataset['test'], Artificial_dataset['anomaly'])
b = sum(Artificial_dataset['anomaly']['start'] == Artificial_dataset['anomaly']['end'])
a = len(Artificial_dataset['anomaly'])
print(a, b, a-b)
with open('processed/datasets/realTweets.pickle', 'wb') as f:
    pickle.dump(Artificial_dataset, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17187838130274832, 'fraction_numerical': 0.062432373945033544, 'mean_word_length': 4.031301034295046, 'pattern_counts': {'":': 9, '<': 36, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '437693', 'license': 'MIT', 'n_tokens_mistral': 7554, 'n_tokens_neox': 6724, 'n_words': 1211, 'path': 'notebooks/1.1-lw-data.ipynb', 'repo_name': 'lcwong0928/hitlads', 'size': '18484'}