<jupyter_start><jupyter_text>Sentiment analysis  Carga de datos<jupyter_code>%matplotlib inline

import spacy
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from json import loads as json_loads
c = ['date', 'file', 'date2', 'log', 'app', 'beat', 'front', 'is_log', 'msg', 'offset', 'arch']

data = pd.read_csv('../data/input-logs-example-file.csv', names=c)
data['beat_json'] = data['beat'].apply(lambda x: json_loads(x))
data['hostname'] = data['beat_json'].apply(lambda x: x['hostname'])
data['name'] = data['beat_json'].apply(lambda x: x['name'])
data['version'] = data['beat_json'].apply(lambda x: x['version'])
#data.head()
nlp = spacy.load('en')
def tokenize(msg):
    return [token.lower_ for token in nlp(msg) if not token.is_punct | token.is_space]  
origen = data['msg'].apply(tokenize)
df_words = pd.DataFrame(origen)
df_words.columns = ['words']
df_words.head(20)<jupyter_output><empty_output><jupyter_text>Carga del diccionario por defecto  Diccionario SentiWordNet SentiWordNet 3.0 by <NAME>, <NAME>, and <NAME> (http://sentiwordnet.isti.cnr.it)<jupyter_code>dic_file = "../data/vocab/SentiWordNet_3.0.0_20130122.txt"

senti_raw = pd.read_csv(
    dic_file, 
    sep="\t", 
    names=["POS", "ID", "PosScore", "NegScore", "SynsetTerms", "Gloss"], 
    skiprows=27,
    skipfooter=1,
    engine='python'
)
senti_raw = senti_raw[(senti_raw['PosScore'] + senti_raw['NegScore']) != 0]

assimetric_terms = senti_raw[(senti_raw['PosScore']==1) | (senti_raw['NegScore']==1)]
senti_raw.head()
import re

exploded = senti_raw['SynsetTerms'].apply(lambda x: re.split("\s+", x)).apply(pd.Series)
exploded.head()
senti_exploded = pd.concat([senti_raw[["PosScore", "NegScore"]], exploded], axis=1)
senti_norm = pd.melt(senti_exploded, id_vars=["PosScore", "NegScore"], value_vars=range(0,25))
senti_norm = senti_norm[senti_norm['value'].notnull()]
senti_norm['value'] = senti_norm['value'].apply(lambda x: re.sub("#\d+", "", x))
# senti_norm.sort_values(by='value')
df_senti = senti_norm.groupby("value").mean()
df_senti.info()
df_senti['NegScore'] = df_senti['NegScore'] * -1

df_senti.head()<jupyter_output><empty_output><jupyter_text>Modelo multi diccionario<jupyter_code>def get_model(vocab):
    def _app(words_list):
        dfw = pd.DataFrame(words_list)
        dfw.columns = ['words']

        dfw = dfw.set_index('words')
        dfw = dfw.join(vocab, how='left').fillna(0)

        return dfw.apply(sum, axis=1).sum()
    return _app
senti_model = get_model(df_senti)

senti_scores = df_words['words'].apply(senti_model)
msgs_sent_score = pd.concat([data['msg'], senti_scores], axis=1)
msgs_sent_score.columns = ['msg', 'senti']
msgs_sent_score.head()
df_boxplot = msgs_sent_score[["senti"]]
melted_box = pd.melt(df_boxplot, value_vars=["senti"])

sns.boxplot(x="variable", y="value", data=melted_box, palette="PRGn")
plt.show()
sort_senti = msgs_sent_score.sort_values(by="senti").reindex()

plt.subplots(figsize=(20,10))

plt.subplot(121)
plt.plot(msgs_sent_score.index.tolist(), sort_senti['senti'])

plt.subplot(122)
p1=sns.kdeplot(msgs_sent_score['senti'], shade=True, color="r")

plt.show()
len(msgs_sent_score[msgs_sent_score['senti'] < 0]['msg'].tolist())
test_msg = msgs_sent_score[msgs_sent_score['senti'] > 0]['msg'][5]

test_words = tokenize(test_msg)
scored = map(lambda x: (x, senti_model([x])), test_words)
sorted_words = sorted(scored, key=lambda x: x[1], reverse=True)
filtered_words = filter(lambda x: x[1] != 0, sorted_words)
list(filtered_words)
test_msg = msgs_sent_score[msgs_sent_score['senti'] < 0]['msg'][1]

test_words = tokenize(test_msg)
scored = map(lambda x: (x, senti_model([x])), test_words)
sorted_words = sorted(scored, key=lambda x: x[1], reverse=True)
filtered_words = filter(lambda x: x[1] != 0, sorted_words)
list(filtered_words)<jupyter_output><empty_output><jupyter_text>Freq analysis<jupyter_code>from collections import Counter

x = data['msg'].apply(lambda x: Counter(x.lower()))
letters = x.apply(pd.Series)

letters.head()
def freq_model(msgs):
    letters = msgs.apply(lambda x: Counter(x.lower())).apply(pd.Series)
    stats = pd.DataFrame([letters.mean(), letters.std()]).T.dropna()
    stats.columns = ["mean", "std"]
    stats['min'] = stats['mean'] - stats['std']
    stats['max'] = stats['mean'] + stats['std']
    return stats

letter_space = freq_model(data['msg'])
letter_space.head()
letter_space = letter_space[letter_space['min'] != letter_space['max']]
def render_letter_space(space):
    plt.subplots(figsize=(20, 10))
    sp = space.reset_index()
    sp.columns=['char', 'mean', 'std', 'min', 'max']
    sp = sp.fillna(0)
    sp = sp.sort_values('mean', ascending=False)
    sp = sp.reset_index()[['char', 'mean', 'std']]
    plt.xticks(sp.index, sp.char.values.tolist())
    plt.errorbar(sp.index, sp['mean'], yerr=sp['std'], fmt='o')

render_letter_space(letter_space)
plt.show()
def freq_model(in_col):
    char_df = in_col.apply(lambda x: Counter(x.lower())).apply(pd.Series).fillna(0).T
    cols = char_df.columns.tolist()
    char_df = letter_space[['min', 'max']].join(char_df, how="left")
    char_df['f'] = char_df['max'] - char_df['min']
    for c in cols:
        char_df[c] = char_df[c] - char_df['min']
        char_df[c] = char_df[c] / char_df['f']
    return char_df[cols]
char_df = freq_model(data['msg'])
char_df.head()
def plot_vs_letter_space(letter_space, data):
    #curr_msg = freq_model(selector)
    curr_msg = data.apply(lambda x: Counter(x.lower())).apply(pd.Series).fillna(0).T
    curr_msg = curr_msg.dropna()
    
    render_letter_space(letter_space)
    plt.ylim(-5, 45)
    sns.stripplot(x=curr_msg.index, y=curr_msg[1], color='red')
    plt.show()
    
    render_letter_space(letter_space)
    plt.ylim(-5, 45)
    sns.stripplot(x=curr_msg.index, y=curr_msg[5], color='green')
    plt.show()

plot_vs_letter_space(letter_space, data['msg'])<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>anomalies = char_df.applymap(lambda x : 1 if x > 1 or x < 0 else 0).mean()
msgs_sent_score['freq'] = anomalies

msgs_sent_score.describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18238172920065251, 'fraction_numerical': 0.011582381729200652, 'mean_word_length': 5.95124716553288, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '315515', 'license': 'Apache-2.0', 'n_tokens_mistral': 2462, 'n_tokens_neox': 2396, 'n_words': 390, 'path': 'security-anomalies-logs-data/notebooks/.ipynb_checkpoints/Log Mining-checkpoint.ipynb', 'repo_name': 'cr0hn/Tarkin', 'size': '6162'}