<jupyter_start><jupyter_text>Spam Classification (NLP)<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!cp ./drive/My\ Drive/Day\ 27/*.csv .<jupyter_output><empty_output><jupyter_text>Data Loading<jupyter_code>import pandas as pd
df = pd.read_csv('spam.csv',encoding='latin-1')
df.head()
df.columns = "label text t1 t2 t2".split()
df.head()
import matplotlib.pyplot as plt
%matplotlib inline

spam_count = df.groupby('label').count()

plt.bar(spam_count.index.values, spam_count['text'])
plt.xlabel('Spam')
plt.ylabel('Count')<jupyter_output><empty_output><jupyter_text>Data Cleansing<jupyter_code>import nltk
nltk.download("stopwords")<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.<jupyter_text>Removing Stopwords<jupyter_code>stop_words = nltk.corpus.stopwords.words("english")

text = df.text
text.head()
# Stop word delim
text = text.apply(lambda x: ' '.join(term for term in x.split() if term not in set(stop_words)))
text.head()<jupyter_output><empty_output><jupyter_text>Masking<jupyter_code>text = text.str.replace(r'\b[\w\-.]+?@\w+?\.\w{2,4}\b' , 'emailadd')#email regex
text = text.str.replace(r'(http[s]?\S+) | (\w+\.[A-Za-z]{2,4}\S*)' , 'httpaddr') # url regex
text = text.str.replace(r'\b(\+\d{1,2}\s)?\d?[\-(.]\d{3}\)?\d{3}[\s.-]?\d{4}\b','phone') #phone regex
text = text.str.replace(r'\d+(\.\d+)?','numbr')    # number regex
text.head()<jupyter_output><empty_output><jupyter_text>Delete spaces<jupyter_code>text = text.str.lower() # to lower case
text = text.str.replace(r'[^\w\d\s]',' ')
text = text.str.replace(r'\s+',' ')
text = text.str.replace(r'^\s+|\s+?$','')
text.head()<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>porter = nltk.PorterStemmer()

df.text = text.apply(lambda x: ' '.join([porter.stem(term) for term in x.split()]))
df.head()<jupyter_output><empty_output><jupyter_text>Feature Engieering  Vektorisasi<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import RegexpTokenizer

token = RegexpTokenizer(r'[a-zA-Z0-9]+')
cv = CountVectorizer(lowercase=True, stop_words='english', ngram_range=(1,1), tokenizer=token.tokenize)
text1 = cv.fit_transform(text)
text1
text2 = text.apply(lambda x: ' '.join([porter.stem(term) for term in x.split()] ))
text2 = cv.fit_transform(text2)
text2<jupyter_output><empty_output><jupyter_text>Feature Extraction<jupyter_code>s = ' '.join(df.text)
print(s)
str_list = s.lower().split()
print(len(str_list))
unique_words = set(str_list)
print(len(unique_words))
print(unique_words)
dictwords = dict()
for word in unique_words:
    dictwords[word] = [str_list.count(word)]
dataham = df[df.label == 'ham']
dataspam = df[df.label == 'spam']
sham = ' '.join(dataham.text)
sspam = ' '.join(dataspam.text)
sham_list = sham.lower().split()
sspam_list = sspam.lower().split()
for words in unique_words:
    iham = sham_list.count(words)
    ispam = sspam_list.count(words)

    dictwords[words].append(iham)
    dictwords[words].append(ispam)
    dictwords[words].append(ispam-iham)
dictwords['number']
sortfreq = sorted(dictwords.items(), key=lambda x: x[1][3], reverse=True)
dictfreq = sortfreq[:100]
print(len(dict(dictfreq)))
dictfreq<jupyter_output>100<jupyter_text>Excercise 1manual classification<jupyter_code>nltk.download('punkt')<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.<jupyter_text>Manual prediction function<jupyter_code>from nltk import word_tokenize

dictfreq = dict(dictfreq)

def pred(text):
    words = list(word_tokenize(text))
    spam = 0
    ham = 0
    for word in words:
        if word in dictfreq.keys():
            spam += dictfreq[word][2]
            ham += dictfreq[word][1]

    return 'spam' if spam > ham else 'ham'
text = df[df.label == 'spam'].values[2]
label = 'spam'

print(text[1])
print(label)
print(pred(text[1]))<jupyter_output>winner as valu network custom select receivea å numbr prize reward to claim call numbr claim code klnumbr valid numbr hour onli
spam
spam<jupyter_text>Sampling<jupyter_code>samples = df.sample(frac=0.75)
spam_count = samples.groupby('label').count()
print(spam_count)<jupyter_output>text  t1  t2  t2
label                  
ham    3627  37   9   5
spam    552   4   2   0<jupyter_text>Manual accuracy<jupyter_code>import numpy as np

def check_acc(samples):
    true_pred = 0
    num_samples = len(samples.index)
    for _, sample in samples.iterrows():
        if pred(sample.text) == sample.label:
            true_pred += 1

    return (true_pred/num_samples)

print('accuracy from manual method with 75% sample is', check_acc(samples))<jupyter_output>accuracy from manual method with 75% sample is 0.7707585546781527<jupyter_text>Machine Learning<jupyter_code>data_text = cv.fit_transform(df.text)
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(data_text, df.label, test_size = 0.10, random_state = 1)
print(type(data_text))
from sklearn.naive_bayes import MultinomialNB

model = MultinomialNB()
model.fit(x_train,y_train)

model.score(x_test, y_test)
from sklearn.svm import SVC

model = SVC(gamma="scale")
model.fit(x_train,y_train)

model.score(x_test, y_test)
from sklearn.linear_model import LogisticRegressionCV

model = LogisticRegressionCV(cv=10)
model.fit(x_train,y_train)

model.score(x_test, y_test)<jupyter_output><empty_output><jupyter_text>Text Clustering<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

documents = ["This little kitty came to play when I was eating at a restaurant.",
             "Merley has the best squooshy kitten belly.",
             "Google Translate app is incredible.",
             "If you open 100 tab in google you get a smiley face.",
             "Best cat photo I've ever taken.",
             "Climbing ninja cat.",
             "Impressed with google map feedback.",
             "Key promoter extension for Google Chrome"
             ]
vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(documents)

true_k = 2
model = KMeans(n_clusters=true_k, init='k-means++',max_iter=100, n_init=1)
model.fit(X)
print("Top terms per cluster:")
order_centroids = model.cluster_centers_.argsort()[:,::-1]
terms = vectorizer.get_feature_names()

for i in range(true_k):
    print("Cluster %d:"%i)
    for ind in order_centroids[i,:10]:
        print(' %s'%terms[ind])

print("\n")
print("prediction")

Y = vectorizer.transform(["chrome browser to open"])
prediction = model.predict(Y)
print(prediction)

Y = vectorizer.transform(["My cat is hungry"])
prediction = model.predict(Y)
print(prediction)<jupyter_output>Top terms per cluster:
Cluster 0:
 google
 translate
 app
 feedback
 impressed
 map
 incredible
 extension
 promoter
 chrome
Cluster 1:
 cat
 best
 climbing
 ninja
 ve
 photo
 taken
 belly
 merley
 kitten


prediction
[0]
[1]<jupyter_text>NN for Text classification  MLP<jupyter_code>from sklearn.preprocessing import LabelEncoder
lb= LabelEncoder()
df["label_code"] = lb.fit_transform(df.label)
x_train, x_test, y_train, y_test = train_test_split(data_text, df.label_code, test_size = 0.10, random_state = 1)
from keras.models import Sequential
from keras import layers

input_dim = x_train.shape[1]
model = Sequential()
model.add(layers.Dense(10, input_dim=input_dim, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
history = model.fit(x_train, y_train, batch_size=32, epochs=10,verbose=1, validation_split=0.1)
loss, accuracy = model.evaluate(x_train,y_train, verbose=True)
print("Training accuracy: {:.4f} with loss: {:.4f}".format(accuracy, loss))
loss, accuracy = model.evaluate(x_test,y_test, verbose=True)
print("Testing accuracy: {:.4f} with loss: {:.4f}".format(accuracy, loss))<jupyter_output>558/558 [==============================] - 0s 71us/step
Testing accuracy: 0.9875 with loss: 0.0464<jupyter_text>LSTM<jupyter_code>from keras.layers import Embedding, Bidirectional, LSTM, Dropout, Dense

model2 = Sequential()
model2.add(Embedding(len(unique_words), 50))
model2.add(Bidirectional(LSTM(32)))
model2.add(Dense(1, activation='sigmoid'))

model2.compile(loss="binary_crossentropy", optimizer='adam',metrics=['accuracy'])
history = model2.fit(x_train, y_train, batch_size=100, epochs=3,verbose=1, validation_split=0.1)
loss, accuracy = model2.evaluate(x_train,y_train, verbose=True)
print("Training accuracy: {:.4f} with loss: {:.4f}".format(accuracy, loss))
loss, accuracy = model2.evaluate(x_test,y_test, verbose=True)
print("Testing accuracy: {:.4f} with loss: {:.4f}".format(accuracy, loss))<jupyter_output>558/558 [==============================] - 57s 102ms/step
Testing accuracy: 0.8781 with loss: 0.3726
{'fraction_non_alphanumeric': 0.14940350094770877, 'fraction_numerical': 0.02096108819266362, 'mean_word_length': 5.679076693968727, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '38831', 'license': 'MIT', 'n_tokens_mistral': 3426, 'n_tokens_neox': 3269, 'n_words': 649, 'path': 'D27_JunaediFahmi.ipynb', 'repo_name': 'juunnn/DTS-AI-2019', 'size': '8969'}