<jupyter_start><jupyter_text>1 Data 1.1 Extract dataThe dataset is from https://www.kaggle.com/boltzmannbrain/nab In realKnownCause/ambient_temperature_system_failure.csv<jupyter_code>df = pd.read_csv("realKnownCause/realKnownCause/ambient_temperature_system_failure.csv")<jupyter_output><empty_output><jupyter_text>1.2 Understand data<jupyter_code>print(df.head())
# check the timestamp format and frequence 
print(df['timestamp'].head(10))
# check the temperature mean
print(df['value'].mean())
print(df.shape)
# change the type of timestamp column for plotting
df['timestamp'] = pd.to_datetime(df['timestamp'])
# change fahrenheit to °C (temperature mean= 71 -> fahrenheit)
df['value'] = (df['value'] - 32) * 5/9
# plot the data
df.plot(x='timestamp', y='value')<jupyter_output><empty_output><jupyter_text>1.3 Feature engineeringExtracting some useful features<jupyter_code># the hours and if it's night or day (7:00-22:00)
df['hours'] = df['timestamp'].dt.hour
df['daylight'] = ((df['hours'] >= 7) & (df['hours'] <= 22)).astype(int)
# the day of the week (Monday=0, Sunday=6) and if it's a week end day or week day.
df['DayOfTheWeek'] = df['timestamp'].dt.dayofweek
df['WeekDay'] = (df['DayOfTheWeek'] < 5).astype(int)
# time with int to plot easily
df['time_epoch'] = (df['timestamp'].astype(np.int64)/100000000000).astype(np.int64)
# Take useful feature and standardize them 
data = df[['value']]#, 'hours', 'daylight', 'DayOfTheWeek', 'WeekDay']]

# for c in ['hours', 'daylight', 'DayOfTheWeek', 'WeekDay']:
# #     print(c)
#     print(f"{c} unique values :{data[c].unique()}")
print(data.shape)
data.head()
from sklearn.neighbors import LocalOutlierFactor

min_max_scaler = preprocessing.StandardScaler()
np_scaled = min_max_scaler.fit_transform(data)
# train one class SVM 
model =  LocalOutlierFactor(n_neighbors=50, novelty=True)
data = pd.DataFrame(np_scaled)

test_items = 3000 ## last how many items

model.fit(data[:data.shape[0]-test_items])
# add the data to the main  


full_span = pd.concat([pd.Series([0 for _ in range(data.shape[0]-test_items)]), pd.Series(model.predict(data[data.shape[0]-test_items: data.shape[0]]))], ignore_index=True)
print(data.shape)
print(full_span.shape)
df['anomaly26'] = full_span
df['anomaly26'] = df['anomaly26'].map( {1: 0, -1: 1} )
print(df['anomaly26'].value_counts())
# visualisation of anomaly throughout time (viz 1)
fig, ax = plt.subplots()

a = df.loc[df['anomaly26'] == 1, ['time_epoch', 'value']] #anomaly

ax.plot(df['time_epoch'], df['value'], color='blue')
ax.scatter(a['time_epoch'],a['value'], color='red')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1731651376146789, 'fraction_numerical': 0.027140672782874617, 'mean_word_length': 6.811940298507463, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '636903', 'license': 'Apache-2.0', 'n_tokens_mistral': 1017, 'n_tokens_neox': 942, 'n_words': 202, 'path': 'Anomaly Detection AI use case/Anomaly detection Tempurature-LOF.ipynb', 'repo_name': 'QaisarRajput/CloudLogs-AnomalyDetection', 'size': '2616'}