<jupyter_start><jupyter_text>N-grams<jupyter_code>#   https://github.com/martin-majlis/Wikipedia-API
import wikipediaapi

wiki_wiki = wikipediaapi.Wikipedia('en')

page = wiki_wiki.page('Natural language processing')
print(page.text)
from nltk.tokenize import word_tokenize

from nltk.corpus import stopwords
stop_words_en = stopwords.words("english")
import string
punctation = list(string.punctuation)
from unidecode import unidecode
from nltk.stem import WordNetLemmatizer

nltk_word_para = word_tokenize(page.summary, language="english", preserve_line=False)

nltk_word_para
def Ngrams_generator(text,ngram=1,isLemma=True):
  words=[unidecode(word).lower() for word in word_tokenize(text, language="english", preserve_line=True) if (word not in stop_words_en)and(word not in punctation) and (word.isalpha())]
  if isLemma:
    lemma = WordNetLemmatizer()
    words=[lemma.lemmatize(w) for w in words]
  temp=zip(*[words[i:] for i in range(0,ngram)])
  ans=[' '.join(ngram) for ngram in temp]
  return ans

print("UNIGRAMS")
unigram = Ngrams_generator(page.summary, 1)
print(unigram)
print("BIGRAMS")
bigram = Ngrams_generator(page.summary, 2)
print(bigram)
print("TRIGRAMS")
trigram = Ngrams_generator(page.summary, 3)
print(trigram)

import pyodbc
import pandas as pd
import pickle

#   conn_str = (
#               "DRIVER={PostgreSQL ODBC Driver(UNICODE)};"
#               "SERVER=localhost;"
#               "DATABASE=NLP;"
#               "UID=postgres;"
#               "PWD=<PASSWORD>;"
#               "PORT=5432;"
#               )
#   pyodbcConn = pyodbc.connect(conn_str)
#   positiveComments = pd.read_sql_query(
#                   f"""
#                   SELECT comment
#                   FROM public.bgg_bgg_15m_reviews
#                   WHERE cast(rating as real)>=10
#                       and comment !='NaN'
#                   LIMIT 1000;
#                   """
#                   , pyodbcConn, index_col=None, coerce_float=True, params=None, parse_dates=None, chunksize=None)
#   
#   positiveComments.to_pickle('positiveComments.pickle')
#   positiveComments
#   negativeComments = pd.read_sql_query(
#                   f"""
#                   SELECT comment
#                   FROM public.bgg_bgg_15m_reviews
#                   WHERE cast(rating as real)<=1
#                       and comment !='NaN'
#                   LIMIT 1000;
#                   """
#                   , pyodbcConn, index_col=None, coerce_float=True, params=None, parse_dates=None, chunksize=None)
#   
#   negativeComments.to_pickle('negativeComments.pickle')
#   negativeComments
positiveComments = pd.read_pickle('positiveComments.pickle')
print(positiveComments.head())

negativeComments = pd.read_pickle('negativeComments.pickle')
print(negativeComments.head())
from nltk.tokenize import word_tokenize
import string
punct = string.punctuation
from nltk.corpus import stopwords
stop_words_en = stopwords.words("english")
from unidecode import unidecode

def removePunctStopWord(sentence):
    nltk_word = word_tokenize(sentence)
    cleanedList = [unidecode(w.lower()) 
                    for w in nltk_word 
                        if (w not in list(string.punctuation)) 
                            and (w not in stop_words_en)]
    cleanedSentence = ' '.join(cleanedList)
    return(cleanedSentence)

positiveComments['comment'] = positiveComments['comment'].apply(lambda x:removePunctStopWord(x))
negativeComments['comment'] = negativeComments['comment'].apply(lambda x:removePunctStopWord(x))

print(positiveComments.head())
print(negativeComments.head())
from collections import defaultdict

positiveValues = defaultdict(int)
negativeValues = defaultdict(int)

#get the count of every word in the positive dataframe
for text in positiveComments.comment:
  for word in Ngrams_generator(text):
    positiveValues[word]+=1

#get the count of every word in the positive dataframe
for text in negativeComments.comment:
  for word in Ngrams_generator(text):
    negativeValues[word]+=1

negativeValues
#focus on more frequently occuring words for every sentiment=>
#sort in DO wrt 2nd column in each of positiveValues,negativeValues and neutralValues
df_positive=pd.DataFrame(sorted(positiveValues.items(),key=lambda x:x[1],reverse=True))
print(df_positive.head())
df_negative=pd.DataFrame(sorted(negativeValues.items(),key=lambda x:x[1],reverse=True))
print(df_negative.head())

pd1=df_positive[0][:10]
pd2=df_positive[1][:10]

ned1=df_negative[0][:10]
ned2=df_negative[1][:10]
import matplotlib.pyplot as plt

plt.figure(1,figsize=(16,4))
plt.bar(pd1,pd2, color ='green',
        width = 0.4)
plt.xlabel("Words in positive dataframe")
plt.ylabel("Count")
plt.title("Top 10 words in positive dataframe-UNIGRAM ANALYSIS")
plt.savefig("positive-unigram.png")
plt.show()
plt.figure(1,figsize=(16,4))
plt.bar(ned1,ned2, color ='red',
        width = 0.4)
plt.xlabel("Words in negative dataframe")
plt.ylabel("Count")
plt.title("Top 10 words in negative dataframe-UNIGRAM ANALYSIS")
plt.savefig("negative-unigram.png")
plt.show()
positiveValues2=defaultdict(int)
negativeValues2=defaultdict(int)

for text in positiveComments.comment:
  for word in Ngrams_generator(text,2):
    positiveValues2[word]+=1

for text in negativeComments.comment:
  for word in Ngrams_generator(text,2):
    negativeValues2[word]+=1

df_positive2=pd.DataFrame(sorted(positiveValues2.items(),key=lambda x:x[1],reverse=True))
pd1bi=df_positive2[0][:10]
pd2bi=df_positive2[1][:10]

df_negative2=pd.DataFrame(sorted(negativeValues2.items(),key=lambda x:x[1],reverse=True))
ned1bi=df_negative2[0][:10]
ned2bi=df_negative2[1][:10]
plt.figure(1,figsize=(16,4))
plt.bar(pd1bi,pd2bi, color ='green',width = 0.4)
plt.xlabel("Words in positive dataframe")
plt.ylabel("Count")
plt.title("Top 10 words in positive dataframe-BIGRAM ANALYSIS")
plt.savefig("positive-bigram.png")
plt.show()
plt.figure(1,figsize=(16,4))
plt.bar(ned1bi,ned2bi, color ='red',
        width = 0.4)
plt.xlabel("Words in negative dataframe")
plt.ylabel("Count")
plt.title("Top 10 words in negative dataframe-BIGRAM ANALYSIS")
plt.savefig("negative-bigram.png")
plt.show()
positiveValues3=defaultdict(int)
negativeValues3=defaultdict(int)

for text in positiveComments.comment:
  for word in Ngrams_generator(text,3):
    positiveValues3[word]+=1

for text in negativeComments.comment:
  for word in Ngrams_generator(text,3):
    negativeValues3[word]+=1

df_positive3=pd.DataFrame(sorted(positiveValues3.items(),key=lambda x:x[1],reverse=True))
pd1tri=df_positive3[0][:10]
pd2tri=df_positive3[1][:10]

df_negative3=pd.DataFrame(sorted(negativeValues3.items(),key=lambda x:x[1],reverse=True))
ned1tri=df_negative3[0][:10]
ned2tri=df_negative3[1][:10]
plt.figure(1,figsize=(16,4))
plt.bar(pd1tri,pd2tri, color ='green',
        width = 0.4)
plt.xticks(rotation=45)
plt.xlabel("Words in positive dataframe")
plt.ylabel("Count")
plt.title("Top 10 words in positive dataframe-TRIGRAM ANALYSIS")
plt.savefig("positive-trigram.png")
plt.show()
plt.figure(1,figsize=(16,4))
plt.bar(ned1tri,ned2tri, color ='red',
        width = 0.4)
plt.xticks(rotation=45)
plt.xlabel("Words in negative dataframe")
plt.ylabel("Count")
plt.title("Top 10 words in negative dataframe-TRIGRAM ANALYSIS")
plt.savefig("negative-trigram.png")
plt.show()<jupyter_output><empty_output><jupyter_text>Skip-grams<jupyter_code>import nltk
text = page.summary

words=[unidecode(word).lower() for word in word_tokenize(text, language="english", preserve_line=True) if (word not in stop_words_en)and(word not in punctation) and (word.isalpha())]
lemma = WordNetLemmatizer()
words=[lemma.lemmatize(w) for w in words]

list(nltk.skipgrams(words, 3, 2))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12886464016627697, 'fraction_numerical': 0.02325279293322941, 'mean_word_length': 4.686115214180207, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '464612', 'license': 'MIT', 'n_tokens_mistral': 2843, 'n_tokens_neox': 2607, 'n_words': 505, 'path': '2021-22/Code_05_NLP_OtherFeatures/05_a_N-grams_NLP_OtherFeatures.ipynb', 'repo_name': 'ranieri-unimi/notes-ferrara-2022', 'size': '7696'}