<jupyter_start><jupyter_text>Naive Bayes Classifier<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Preparing the data<jupyter_code>df=pd.read_csv('movie_data.csv',encoding='utf-8')
df.head()<jupyter_output><empty_output><jupyter_text>Cleansing<jupyter_code>import re
def preprocessor(text):
    text = re.sub('<[^>]*>', '', text)
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)',
                           text)
    text = (re.sub('[\W]+', ' ', text.lower()) +
            ' '.join(emoticons).replace('-', ''))
    return text
df['review']=df['review'].apply(preprocessor)
#from nltk.stem.porter import PorterStemmer
#def tokenizer_porter(text):
#    porter=PorterStemmer()
#    return [porter.stem(word) for word in text.split()]
#df['review']=df['review'].apply(tokenizer_porter)
#from nltk.corpus import stopwords
#stop=stopwords.words('english')
#def stop_word_removal(text):
#    return [w for w in text if w not in stop]
#df['review']=df['review'].apply(stop_word_removal)
df.head()<jupyter_output><empty_output><jupyter_text>BoW (Bag-of-Words)<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
count=CountVectorizer()
bag=count.fit_transform(df.review)
bag_train=bag[:30000]
bag_validation=bag[30000:40000]
bag_test=bag[40000:]
bag_train_pos=bag_train[(df[:30000].sentiment.values==1)]
bag_train_neg=bag_train[(df[:30000].sentiment.values==0)]
bag_validation_pos=bag_validation[(df[30000:40000].sentiment.values==1)]
bag_validation_neg=bag_validation[(df[30000:40000].sentiment.values==0)]
bag_test_pos=bag_test[(df[40000:].sentiment.values==1)]
bag_test_neg=bag_test[(df[40000:].sentiment.values==0)]<jupyter_output><empty_output><jupyter_text>Training (Maximum likelihood estimation)<jupyter_code>#theta (smoothing)
gamma=2
theta_pos=(bag_train_pos.sum(axis=0)+gamma)/(bag_train_pos.sum()+gamma)
theta_neg=(bag_train_neg.sum(axis=0)+gamma)/(bag_train_neg.sum()+gamma)
#coefficient vector
alpha=np.log(theta_pos/theta_neg)<jupyter_output><empty_output><jupyter_text>Training (Determine threshold value)<jupyter_code>a_validation_pos=[]
for i in range(bag_validation_pos.shape[0]):
    a_validation_pos.append((alpha*bag_validation_pos[i].T)[0,0])
a_validation_neg=[]
for i in range(bag_validation_neg.shape[0]):
    a_validation_neg.append((alpha*bag_validation_neg[i].T)[0,0])
plt.hist(a_validation_pos,bins=100,alpha=0.5,label='anomaly score(positive)');
plt.hist(a_validation_neg,bins=100,alpha=0.5,label='anomaly score(negative)');
plt.legend();
nsa=[] #normal sample accuracy
asa=[] #anomalous sample accuracy
for threshold in range(-150,150,1):
    nsa.append(1-((np.array(a_validation_pos)<threshold).sum())/len(a_validation_pos))
    asa.append(((np.array(a_validation_neg)<threshold).sum())/len(a_validation_neg))
plt.plot(range(-150,150,1),nsa,label='normal sample accuracy')
plt.plot(range(-150,150,1),asa,label='anomalous sample accuracy')
plt.axvline(-2, ls = "--", color = "navy")
plt.legend();<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>#normal sample accuracy
a_test_pos=[]
for i in range(bag_test_pos.shape[0]):
    a_test_pos.append((alpha*bag_test_pos[i].T)[0,0])
1-((np.array(a_test_pos)<-2).sum()/len(a_test_pos))
#anomalous sample accuracy
a_test_neg=[]
for i in range(bag_test_neg.shape[0]):
    a_test_neg.append((alpha*bag_test_neg[i].T)[0,0])
(np.array(a_test_neg)<-2).sum()/len(a_test_neg)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17779680365296804, 'fraction_numerical': 0.03367579908675799, 'mean_word_length': 8.817927170868348, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '138506', 'license': 'MIT', 'n_tokens_mistral': 1433, 'n_tokens_neox': 1335, 'n_words': 157, 'path': 'Chapter 3 Naive Bayes.ipynb', 'repo_name': 'TakuyaMurata/Anomaly-Detection-and-Change-Detection', 'size': '3504'}