<jupyter_start><jupyter_text>N-Grams, Regex, and TF-IDF You are an analyst working at McDonalds' corporate headquarters, and charged with identifying areas for improvement to increase customer service.Using the `mcdonalds-yelp-negative-reviews.csv` dataset, clean and parse the text reviews. Finally, generate a TF-IDF report that **visualizes** for each city what the major source of complaints with the McDonalds franchises are. Offer your analysis and business recommendations on next steps for the global SVP of Operations.<jupyter_code>import nltk
nltk.download('stopwords')
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
from nltk.collocations import BigramCollocationFinder, BigramAssocMeasures
from nltk.stem import WordNetLemmatizer
from nltk import word_tokenize
lemmatizer = WordNetLemmatizer()
from nltk.tokenize import sent_tokenize
from nltk.stem import PorterStemmer
from nltk.stem import LancasterStemmer
data = pd.read_csv("mcdonalds-yelp-negative-reviews.csv", encoding="latin1")<jupyter_output><empty_output><jupyter_text>explore data<jupyter_code>data.shape
data.head()
# city
city = data['city'].value_counts()
print(city)
print("there are",len(data['city'].value_counts()),"cities")<jupyter_output>Las Vegas      409
Chicago        219
Los Angeles    167
New York       165
Atlanta        130
Houston        105
Portland        97
Dallas          75
Cleveland       71
Name: city, dtype: int64
there are 9 cities<jupyter_text>all reviews<jupyter_code>allreview = list(data["review"].values)
type(allreview)
# word count for all reviews
# to find potential customize stop words 
words = [] 
word_count = {} 

for line in allreview: 
    for word in line.split(" "): 
        words.append(word.lower())
        
        if word not in word_count.keys(): 
            word_count[word] = 1
        else:
            word_count[word] += 1 
import operator
sorted_review = sorted(word_count.items(), key=operator.itemgetter(1),reverse=True)
# lemmatize all review
lemmatizer = WordNetLemmatizer()

def lemma(lines_review):
    sentence1 =[]
    for sentence in lines_review:
        token_words=word_tokenize(sentence)
        token_words
        stem_sentence=[]
        for word in token_words:
            stem_sentence.append(lemmatizer.lemmatize(word))
            stem_sentence.append(" ")
        sentence1.append("".join(stem_sentence))
    return sentence1

allreview = lemma(allreview)

print(allreview[2])
print(type(allreview))
vectorizer = TfidfVectorizer(ngram_range=(2,5),
                             token_pattern=r'\b[a-zA-Z]{3,}\b',
                             max_df=0.5,
                             binary = True,
                             min_df=2, stop_words=stopwords.words('english')+ ['.', ',',"'s", 'wa','', "n't",'...',\
                                                                              'mcdonalds','mcdonald','McDonald','McDonalds',\
                                                                              'one','get','would','could','know','even','got',"fast","food"]) #customize stops words

X = vectorizer.fit_transform(allreview)
terms = vectorizer.get_feature_names()
tf_idf = pd.DataFrame(X.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score = pd.DataFrame(tf_idf, columns=["score"])
score["term"] = terms
score.sort_values(by="score", ascending=False, inplace=True)
score.head(30)<jupyter_output><empty_output><jupyter_text>Las Vegas<jupyter_code># filter only Las Vegas
Vegas = data[data.city == 'Las Vegas']
r_vegas = list(Vegas['review'].values)

# lemmatize all review
lemma(r_vegas)

# vectorizer1 2-grams
vectorizer1 = TfidfVectorizer(ngram_range=(2,5),
                             token_pattern=r'\b[a-zA-Z]{3,}\b',
                             binary = True,
                             max_df=0.3,
                             min_df=2, stop_words=stopwords.words('english')+ ['.', ',',"'s", 'wa','', "n't",'...',\
                                                                              'mcdonalds','mcdonald','McDonald','McDonalds',\
                                                                              'one','get','would','could','know','even','got',"fast","food"]) #customize stops words

# vectorizer2 3-grams
vectorizer2 = TfidfVectorizer(ngram_range=(3,3),
                             token_pattern=r'\b[a-zA-Z]{3,}\b',
                             binary = True,
                             max_df=0.3,
                             min_df=2, stop_words=stopwords.words('english')+ ['.', ',',"'s", 'wa','', "n't",'...',\
                                                                              'mcdonalds','mcdonald','McDonald','McDonalds',\
                                                                              'one','get','would','could','know','even','got',"fast","food"]) #customize stops words

# vectorizer3
vectorizer3 = TfidfVectorizer(ngram_range=(4,4),
                             token_pattern=r'\b[a-zA-Z]{3,}\b',
                             binary = True,
                             max_df=0.3,
                             min_df=1, stop_words=stopwords.words('english')+ ['.', ',',"'s", 'wa','', "n't",'...',\
                                                                              'mcdonalds','mcdonald','McDonald','McDonalds',\
                                                                              'one','get','would','could','know','even','got',"fast","food"]) #customize stops words
# 2-5 grams
vegas = vectorizer1.fit_transform(r_vegas)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(vegas.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)

score1.head(10)
# 3-grams
vegas = vectorizer2.fit_transform(r_vegas)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(vegas.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)

# 4-grams
vegas = vectorizer3.fit_transform(r_vegas)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(vegas.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)<jupyter_output><empty_output><jupyter_text>Chicago<jupyter_code># filter only Chicago
Chicago = data[data.city == 'Chicago']
r_chicago = list(Chicago['review'].values)

# lemmatize review
r_chicago = lemma(r_chicago)
# 2-5 grams
chicago = vectorizer1.fit_transform(r_chicago)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(chicago.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)
# 3-grams
chicago = vectorizer2.fit_transform(r_chicago)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(chicago.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
chicago = vectorizer3.fit_transform(r_chicago)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(chicago.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)
score1.to_csv('chicago.csv')<jupyter_output><empty_output><jupyter_text>Los Angeles<jupyter_code># filter only LA
LA = data[data.city == 'Los Angeles']
r_la = list(LA['review'].values)

# lemmatize review
r_la = lemma(r_la)
# 2-5 grams
la = vectorizer1.fit_transform(r_la)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(la.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)

# 3-grams
la = vectorizer2.fit_transform(r_la)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(la.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
la = vectorizer3.fit_transform(r_la)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(la.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)<jupyter_output><empty_output><jupyter_text>New York<jupyter_code># filter only LA
NY = data[data.city == 'New York']
r_ny = list(NY['review'].values)

# lemmatize review
r_ny = lemma(r_ny)
# 2-5 grams
ny = vectorizer1.fit_transform(r_ny)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(ny.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)
# 3-grams
ny = vectorizer2.fit_transform(r_ny)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(ny.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
ny = vectorizer3.fit_transform(r_ny)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(ny.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)
score1.to_csv('ny.csv')<jupyter_output><empty_output><jupyter_text>Atlanta<jupyter_code># filter only LA
Atlanta = data[data.city == 'Atlanta']
r_atlanta = list(Atlanta['review'].values)

# lemmatize review
r_atlanta = lemma(r_atlanta)
# 2-5 grams
atlanta = vectorizer1.fit_transform(r_atlanta)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(atlanta.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)
# 3-grams
atlanta = vectorizer2.fit_transform(r_atlanta)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(atlanta.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
atlanta = vectorizer3.fit_transform(r_atlanta)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(atlanta.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)
score1.to_csv('altan.csv')<jupyter_output><empty_output><jupyter_text>Houston<jupyter_code># filter only Houston
Houston = data[data.city == 'Houston']
r_houston = list(Houston['review'].values)

# lemmatize review
r_houston = lemma(r_houston)
# 2-5 grams
houston = vectorizer1.fit_transform(r_houston)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(houston.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)
# 3-grams
houston = vectorizer2.fit_transform(r_houston)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(houston.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
houston = vectorizer3.fit_transform(r_houston)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(houston.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)<jupyter_output><empty_output><jupyter_text>Portland<jupyter_code># filter only Portland
Portland = data[data.city == 'Portland']
r_portland = list(Portland['review'].values)

# lemmatize review
r_portland = lemma(r_portland)
# 2-5 grams
portland = vectorizer1.fit_transform(r_portland)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(portland.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)
# 3-grams
portland = vectorizer2.fit_transform(r_portland)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(portland.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
portland = vectorizer3.fit_transform(r_portland)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(portland.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)<jupyter_output><empty_output><jupyter_text>Dallas<jupyter_code># filter only Dallas
Dallas = data[data.city == 'Dallas']
r_dallas = list(Dallas['review'].values)

# lemmatize review
r_dallas = lemma(r_dallas)
# 2-5 grams
dallas = vectorizer1.fit_transform(r_dallas)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(dallas.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)
# 3-grams
dallas = vectorizer2.fit_transform(r_dallas)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(dallas.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
dallas = vectorizer3.fit_transform(r_dallas)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(dallas.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)
score1.to_csv('dallas.csv')<jupyter_output><empty_output><jupyter_text>Cleveland<jupyter_code># filter only Cleveland
Cleveland = data[data.city == 'Cleveland']
r_cleveland = list(Cleveland['review'].values)

# lemmatize review
r_cleveland = lemma(r_cleveland)
# 2-5 grams
cleveland = vectorizer1.fit_transform(r_cleveland)
terms = vectorizer1.get_feature_names()
tf_idf = pd.DataFrame(cleveland.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score1 = pd.DataFrame(tf_idf, columns=["score"])
score1["term"] = terms
score1.sort_values(by="score", ascending=False, inplace=True)

score1.head()
# 3-grams
cleveland = vectorizer2.fit_transform(r_cleveland)
terms = vectorizer2.get_feature_names()
tf_idf = pd.DataFrame(cleveland.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score2 = pd.DataFrame(tf_idf, columns=["score"])
score2["term"] = terms
score2.sort_values(by="score", ascending=False, inplace=True)
# 4-grams
cleveland = vectorizer3.fit_transform(r_cleveland)
terms = vectorizer3.get_feature_names()
tf_idf = pd.DataFrame(cleveland.toarray().transpose(), index=terms)
tf_idf = tf_idf.sum(axis=1)
score3 = pd.DataFrame(tf_idf, columns=["score"])
score3["term"] = terms
score3.sort_values(by="score", ascending=False, inplace=True)<jupyter_output><empty_output><jupyter_text>phrase count for high scored phrases<jupyter_code># count "drive thru"
import re

count = 0
for line in allreview:
    if len(re.findall(r"(drive thru)", line)) >= 1:
        count += 1
count

print("drive thru is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "drive thru" in Vagas
count = 0
for line in r_vegas:
    if len(re.findall(r"(drive thru)", line)) >= 1:
        count += 1

(count/409)*100
# count "order wrong" , "wrong order" , "order right" , "right order" "correct order" "order correct"
count = 0
for line in allreview:
    if len(re.findall(r"(order wrong|wrong order|order right|right order|correct order|order correct)", line)) >= 1:
        count += 1

print("wrong order, order wrong, order right, or right order is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "ice cream"
count = 0
for line in allreview:
    if len(re.findall(r"(ice cream|icecream)", line)) >= 1:
        count += 1
print("ice cream is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "french fry"
count = 0
for line in allreview:
    if len(re.findall(r"(french fry|frenchfry|fries)", line)) >= 1:
        count += 1
print("french fry is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "big mac"
count = 0
for line in allreview:
    if len(re.findall(r"(big mac|bigmac)", line)) >= 1:
        count += 1
print("big mac is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "chicken nugget"
count = 0
for line in allreview:
    if len(re.findall(r"(chicken nugget)", line)) >= 1:
        count += 1
print("chicken nugget is mentioned in",round((count/1525)*100,3), '% of all comments')
#count iced coffee
count = 0
for line in allreview:
    if len(re.findall(r"(iced coffee)", line)) >= 1:
        count += 1
print("ced coffee is mentioned in",round((count/1525)*100,3), '% of all comments')
#count sweet tea
count = 0
for line in allreview:
    if len(re.findall(r"(sweet tea)", line)) >= 1:
        count += 1
print("sweet tea is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "late night"
count = 0
for line in allreview:
    if len(re.findall(r"(late night|latenight)", line)) >= 1:
        count += 1
print("late night is mentioned in",round((count/1525)*100,3), '% of all comments')
# count "parking lot"
count = 0
for line in allreview:
    if len(re.findall(r"(parking lot|parking)", line)) >= 1:
        count += 1
print("parking or parking lot is mentioned in",round((count/1525)*100,3), '% of all comments')<jupyter_output>parking or parking lot is mentioned in 3.148 % of all comments
{'fraction_non_alphanumeric': 0.1498879043450411, 'fraction_numerical': 0.02129817444219067, 'mean_word_length': 4.5265486725663715, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '131620', 'license': 'MIT', 'n_tokens_mistral': 6888, 'n_tokens_neox': 6520, 'n_words': 1221, 'path': 'mcdonalds_yelp_negative_reviews_TFIDF.ipynb', 'repo_name': 'Kailingh/course_works', 'size': '18734'}