<jupyter_start><jupyter_text>NLP on reviews<jupyter_code>#From nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet
import stop_words
import string
from tqdm import tqdm, tqdm_notebook
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
from collections import Counter
import numpy as np
import json

df_rev = pd.read_csv('./data_ratebeer_withReview.csv')
df_rev.head(3)
df_rev.shape
df_des = pd.read_csv('./beer_desc_vocab_clean.csv')
df_des.head(3)<jupyter_output><empty_output><jupyter_text>Text Pre-processing<jupyter_code>stop_words_ = stop_words.get_stop_words("en")
punctuation_filter = str.maketrans({key: None for key in string.punctuation})

def nlp_pre_process(text):
    """
    Reduces an input text into a list of tokens, 
    using NLP filterings such as normalization, 
    stop word filtering.
    text: (str)
    tokens: list(str)
    output: str
    """
    text = text.lower()  # normalize
    text = text.translate(punctuation_filter)  # remove punctuation
    tokens = word_tokenize(text)  # tokenize
    tokens = [t for t in tokens if t not in stop_words_]  # stop words filtering    
    return " ".join(tokens)
tqdm.pandas(tqdm_notebook())
df_rev.reviews = df_rev.reviews.progress_apply(nlp_pre_process)
reviews = df_rev["reviews"]
reviews.tolist()
len(reviews)
vec = CountVectorizer(vocabulary=vocab_list)
vecMat = vec.fit_transform(reviews)
vecMat = vecMat.toarray()
vecMat.shape
df_rev["vect"] = vecMat.tolist()
df_rev.head(3)
def cosBeerJ(df):
    jsonList = dict()
    count = 0
    arr = np.empty((df.shape[0],df.shape[0]))
    arr[:]= np.nan
    for row, name in enumerate(df["name"]):
        count += 1
        if (count%10==0):
            print(count)
        listCos = []
        index = df[df.name == name].index[0]
        for i in range(df.shape[0]):
            if (np.isnan(arr[row,i])):
                result = cosine_similarity(np.asarray(df.loc[index, 'vect']).reshape(1,-1),np.asarray(df.loc[i, 'vect']).reshape(1,-1))
                arr[i,row] = result
                arr[row,i] = result
            else:
                result = arr[row,i]
            listCos.append(result.flatten()[0])
         
        df2 = df.copy()
        df2["sim"] = listCos
        df2 = df2.sort_values(by = "sim",ascending=False)
        df2 = df2.drop("vect", axis = 1)
        preJson = df2[1:6]
        json = preJson.to_json(orient='records')
        jsonList[name] = json
    return jsonList, arr
            
            
            
dfTest = df_rev[["name", "city", "country","vect"]].head(5)
dfTest
dfBeer = df_rev[["name", "city", "country","vect"]]
dfBeer.shape
jsonList, array = cosBeerJ(dfBeer)
array.shape
with open('beerSim.json', 'w') as f:
     json.dump(jsonList, f)
jsonList<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12596833950825193, 'fraction_numerical': 0.009430784776018861, 'mean_word_length': 3.3421052631578947, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '445681', 'license': 'Apache-2.0', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1055, 'n_words': 224, 'path': 'preprocessing/beerSimilarities.ipynb', 'repo_name': 'montalex/COM-480-Data-Visualization', 'size': '2969'}