<jupyter_start><jupyter_text>Anomaly Detection Detecting anomaly in tweet frequencies. I will look at a few examples from the 'realtweets' dataset.<jupyter_code>import pandas as pd
import numpy as np
df = pd.read_csv('../data/external/realTweets/Twitter_volume_GOOG.csv')
df.shape
df.head(5)
# graph data
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

df['datetime'] = pd.to_datetime(df['timestamp'],
                                   format="%Y.%m.%d %H:%M:%S.%f")

df.set_index('datetime', inplace=True)

ax = df['value'].plot()

ticklabels = df.index.strftime('%Y-%m-%d')
ax.xaxis.set_major_formatter(ticker.FixedFormatter(ticklabels))
ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%.2f'))
plt.show()
df.head(5)
#split datetime into hour minute second values
import datetime
df['datetime'] = pd.to_datetime(df['timestamp'])
df['hour'], df['minute'], df['second'] = df['datetime'].dt.hour, df['datetime'].dt.minute, df['datetime'].dt.second
df.head(5)
from sklearn.model_selection import train_test_split
decision = df[['value']]
features = df[['hour', 'minute', 'second']]
train, test, train_d, test_d = train_test_split(features,
                                                decision,
                                                test_size = 0.2,
                                                random_state = 14)
# isolation forest for finding anomalies
from sklearn.ensemble import IsolationForest
clf = IsolationForest(max_samples=100, random_state=14)
clf.fit(train)

# predictions
y_pred_train = clf.predict(train)
y_pred_test = clf.predict(test)
print("Accuracy:", list(y_pred_test).count(1)/y_pred_test.shape[0])<jupyter_output>Accuracy: 0.9062795834648154
{'fraction_non_alphanumeric': 0.13607038123167156, 'fraction_numerical': 0.0187683284457478, 'mean_word_length': 3.6868131868131866, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '564605', 'license': 'MIT', 'n_tokens_mistral': 574, 'n_tokens_neox': 539, 'n_words': 112, 'path': 'notebooks/Anomalies_Twitter.ipynb', 'repo_name': 'patrickcanny/anomaly_detect_tweets', 'size': '1705'}