<jupyter_start><jupyter_text><jupyter_code>#Download IMDB to the current folder
!wget http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz
# Unzip
!tar zxf aclImdb_v1.tar.gz
# aclImdb / train / unsup is unlabeled and removed
!rm -rf aclImdb/train/unsup
# Show IMDB dataset description
!cat aclImdb/README
from sklearn.datasets import load_files
train_review = load_files('./aclImdb/train/', encoding='utf-8')
x_train, y_train = train_review.data, train_review.target
test_review = load_files('./aclImdb/test/', encoding='utf-8')
x_test, y_test = test_review.data, test_review.target
# Display of the correspondence between 0, 1 of the label and the meaning
print(train_review.target_names)
print("x : {}".format(x_train[0]))
mini_dataset = \
  ["This movie is very good.",
  "This film is a good",
  "Very bad. Very, very bad."]
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(token_pattern=r'(?u)\b\w+\b')
bow = (vectorizer.fit_transform(mini_dataset)).toarray()
# Put together in DataFrame
df = pd.DataFrame(bow, columns=vectorizer.get_feature_names())
display(df)
# Specify the range of n-gram used in ngram_range
vectorizer = CountVectorizer(ngram_range=(2, 2), token_pattern=r'(?u)\b\w+\b')
bow_train = (vectorizer.fit_transform(mini_dataset)).toarray()
df = pd.DataFrame(bow_train, columns=vectorizer.get_feature_names())
display(df)<jupyter_output><empty_output><jupyter_text>Scratch implementation of BoW<jupyter_code>first_line = 'This movie is SOOOO funny!!!'.lower().replace('!', '').split()
second_line = 'What a movie! I never'.lower().replace('!', '').split()
third_line = 'best movie ever!!!!! this movie'.lower().replace('!', '').split()
gram_1_feature_names = first_line + second_line + third_line
import numpy as np

gram_1 = pd.DataFrame(np.zeros((3, len(set(gram_1_feature_names)))).astype('int'), columns=set(gram_1_feature_names))

for i, ss in enumerate([first_line, second_line, third_line]):
    for s in ss:
        n = ss.count(s)
        gram_1[s][i] = n

gram_1
gram_2_feature_names = []
gram_2_lines = []
for i, s in enumerate([first_line, second_line, third_line]):
  line = []
  for ss in range(len(s)-1):
    line.append(f'{s[ss]} {s[ss+1]}')
    gram_2_feature_names.append(f'{s[ss]} {s[ss+1]}')
  gram_2_lines.append(line)
gram_2 = pd.DataFrame(np.zeros((3, len(set(gram_2_feature_names)))).astype('int'), columns=set(gram_2_feature_names))

for i, ss in enumerate(gram_2_lines):
    for s in ss:
        n = ss.count(s)
        gram_2[s][i] = n
        
gram_2<jupyter_output><empty_output><jupyter_text>IDF<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
n_samples = 25000
idf = np.log(n_samples/np.arange(1,n_samples))
plt.title("IDF")
plt.xlabel("df(t)")
plt.ylabel("IDF")
plt.plot(idf)
plt.show()
vectorizer = CountVectorizer(stop_words=["is"], token_pattern=r'\b\w+\b')
bow_train = (vectorizer.fit_transform(mini_dataset)).toarray()
df = pd.DataFrame(bow_train, columns=vectorizer.get_feature_names())
display(df)
# Download Stopword for the first time
import nltk
stop_words = nltk.download('stopwords')
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
print("stop word : {}".format(stop_words)) # 'i', 'me', 'my', ...
vectorizer = CountVectorizer(token_pattern=r'\b\w+\b', max_features = 5)
bow_train = (vectorizer.fit_transform(mini_dataset)).toarray()
df = pd.DataFrame(bow_train, columns=vectorizer.get_feature_names())
display(df)<jupyter_output><empty_output><jupyter_text>TF-IDF calculation<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(stop_words= stop_words, max_features=5000)
X_train = vectorizer.fit_transform(x_train)
X_test = vectorizer.fit_transform(x_test)
print(X_train.shape, X_test.shape)<jupyter_output>(25000, 5000) (25000, 5000)<jupyter_text>Learning using TF-IDF<jupyter_code>import lightgbm as lgb

lgb = lgb.LGBMClassifier().fit(X_train,y_train)
y_pred = lgb.predict(X_test)
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
print("{}".format(lgb.score(X_test, y_test)))
print("{}".format(precision_score(y_test,y_pred)))
print("{}".format(recall_score(y_test,y_pred)))
print("{}".format(f1_score(y_test,y_pred)))
print(confusion_matrix(y_test, y_pred))<jupyter_output>0.57548
0.5735672514619883
0.58848
0.5809279368213228
[[7031 5469]
 [5144 7356]]<jupyter_text>Scratch mounting of TF-IDF<jupyter_code>from gensim.models import Word2Vec
sentences = [['this', 'movie', 'is', 'very', 'good'], ['this', 'film', 'is', 'a', 'good'], ['very', 'bad', 'very', 'very', 'bad']]
model = Word2Vec(min_count=1, size=10) # Set the number of dimensions to 10
model.build_vocab(sentences) # Preparation
model.train(sentences, total_examples=model.corpus_count, epochs=model.iter) #Learning

print("List of vocabulary : {}".format(model.wv.vocab.keys()))
for vocab in model.wv.vocab.keys():
  print("Vector of {}: \n{}".format(vocab, model.wv[vocab]))
model.wv.most_similar(positive="good", topn=3)
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
vocabs = model.wv.vocab.keys()
tsne_model = TSNE(perplexity=40, n_components=2, init="pca", n_iter=5000, random_state=23)
vectors_tsne = tsne_model.fit_transform(model[vocabs])
fig, ax = plt.subplots(figsize=(5,5))
ax.scatter(vectors_tsne[:, 0], vectors_tsne[:, 1])
for i, word in enumerate(list(vocabs)):
    plt.annotate(word, xy=(vectors_tsne[i, 0], vectors_tsne[i, 1]))
ax.set_yticklabels([])
ax.set_xticklabels([])
plt.show()<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:5: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).
  """<jupyter_text>Distributed representation of IMDB movie review datasets<jupyter_code>model_2 = Word2Vec(min_count=1, size=10) # Set the number of dimensions to 10
model_2.build_vocab(x_train) # Preparation
model_2.train(x_train, total_examples=model_2.corpus_count, epochs=model_2.iter) #Learning<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: DeprecationWarning: Call to deprecated `iter` (Attribute will be removed in 4.0.0, use self.epochs instead).
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>Corpus pretreatment<jupyter_code>with_url = 0
for i, s in enumerate(x_train):
    if 'www' in s:
      with_url = i
      print('before processing')
      print('-----')
      print(s)
      break
import re
no_preprocessing = x_train[with_url]
after_preprocessing1 = re.sub(r'https?://[\w/:%#\$&\?\(\)~\.=\+\-…]+', "", no_preprocessing) 
after_preprocessing2 = re.sub(r'<[^>]+>', " ", after_preprocessing1) 
after_preprocessing3 = re.sub(r"[^0-9a-zA-Z ]", "", after_preprocessing2) 
after_preprocessing = after_preprocessing3.lower() 
print('after processing')
print('-----')
print(after_preprocessing)<jupyter_output>after processing
-----
i dont hand out ones often but if there was ever a film that deserved this sort of attention its gas this is selfindulgent crap that reaches for some of the ambiance of mash and falls completely flat on its face in the attempt  i see what corman was going for  malcolm marmorstein and <NAME> tried to reproduce goulds deathless role in the original movie version of mash with a similar plot in the movie whiffs  look it up here in imdb  for more information  marmorstein and gould got closer to the brass ring with whiffs than corman did with gas but didnt quite get there neither one of those films even got close to the success of mash  whats wrong with gas what isnt no one comes close to really acting at a level above junior high school theatrics the production values stink someone else here mentioned the magically regenerating headlights on a getaway car and theres more of that lack of attention to detail nothing works the way its supposed to in this film and [...]<jupyter_text>Learning Word2Vec<jupyter_code>model = Word2Vec(min_count=1, size=10) 
model.build_vocab(after_preprocessing) 
model.train(after_preprocessing, total_examples=model.corpus_count, epochs=model.iter)<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: DeprecationWarning: Call to deprecated `iter` (Attribute will be removed in 4.0.0, use self.epochs instead).
  This is separate from the ipykernel package so we can avoid doing imports until
{'fraction_non_alphanumeric': 0.13247210804462714, 'fraction_numerical': 0.022196124486200824, 'mean_word_length': 6.162321278385198, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '481636', 'license': 'Apache-2.0', 'n_tokens_mistral': 3133, 'n_tokens_neox': 2987, 'n_words': 722, 'path': 'Natural_Language_Processing.ipynb', 'repo_name': 'ibrahimsesay/griddb', 'size': '8522'}