<jupyter_start><jupyter_text>Sentiment modelling now using pos_neg column    Negative reviews:      1-3 Stars  = 0    Positive reviews:      4-5 Stars  = 1<jupyter_code>df['pos_neg']=[1 if x>3 else 0 for x in df.overall]
df.tail(5)
#fetching locations of all those pos_negatives as 0 where review is negative
df.loc[df['overall'] < 4, 'pos_neg']
#SERIES
review_text = df["reviewText"]
#Doing train and test split with no random seed
x_train, x_test, y_train, y_test = train_test_split(df.reviewText, df.pos_neg, random_state=0)
print("x_train shape: {}".format(x_train.shape), end='\n')
print("y_train shape: {}".format(y_train.shape), end='\n\n')
print("x_test shape: {}".format(x_test.shape), end='\n')
print("y_test shape: {}".format(y_test.shape), end='\n\n')
#applying logistic regression which is countVectorizer
#min_df ( = 5): defines the minimum frequency of a word for it to be counted as a feature
#ngram- tells the min and max length of sequence of tokens to be considered together---e.g but the, wise man, etc. 
#one gram- unigram, two gram, etc.
#my name, name is, is hardik, <NAME>
# in bigram 2: 1 mapping for key value pair---- i.e. to predict one word if using 2 words then bigram
# choose based on best accuracy among unigram,bigram, etc.
#This fit is not actual train
#  columns	 My, name, is, Hardik, Goel, Aakash, Aggarwal
#  pos_neg
#  1,1,1,1,1,0,0	
#  1,1,1,0,0,1,1	
# countVectorizer- only counts word frequencies whereas tf idf(term frequency inverse document frequency) also takes 
# into consideration most frequently used word(most weighted word) using decimal values
#repr is like string to create unambigous values- used for debug whereas str is used for readability


from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
# Vectorize X_train
vectorizer = CountVectorizer(min_df=5,ngram_range=(2, 2)).fit(x_train)
X_train = vectorizer.transform(x_train)
print("X_train:\n{}".format(repr(X_train)))
print("Vocabulary size: {}".format(len(vectorizer.vocabulary_)))
feature_names = vectorizer.get_feature_names()
print("Number of features: {}".format(len(feature_names)))
#Displaying all the dictionary's words and their frequencies as present in this
vectorizer.get_feature_names()
from sklearn.linear_model import LogisticRegression
#doing 5 fold cross validation
scores = cross_val_score(LogisticRegression(), X_train, y_train, cv=5)
print("CV accuracy: {}".format(np.mean(scores)))
#using an okayish regularization
logreg = LogisticRegression(C=0.1).fit(X_train, y_train)

X_test = vectorizer.transform(x_test)

log_y_pred = logreg.predict(X_test)
logreg_score = accuracy_score(y_test, log_y_pred)
print("Accuracy:   {}".format(logreg_score))
print("Training set score: {}".format(logreg.score(X_train, y_train)))
print("Test set score: {}".format(logreg.score(X_test, y_test)))
log_cfm = confusion_matrix(y_test, log_y_pred)
print("Confusion matrix:")
print(log_cfm, end='\n\n')
print('-'*15)
print(np.array([['TN', 'FP'],[ 'FN' , 'TP']]))
import itertools
plt.imshow(log_cfm, interpolation='nearest')
for i, j in itertools.product(range(log_cfm.shape[0]), range(log_cfm.shape[1])):
    plt.text(j, i, log_cfm[i, j],
             horizontalalignment="center",
             color="brown")

plt.ylabel('True label (Recall)')
plt.xlabel('Predicted label (Precision)')
plt.title('Logistic Reg | Confusion Matrix')
plt.colorbar();
log_f1 = f1_score(y_test, log_y_pred)
print("Logistic Reg - F1 score: {}".format(log_f1))
# Defining a grid of parameters that we would want to test out in the model and select the best model using GridSearchCV
# for the above mentioned hyperparameters.
param_grid = {'C': [0.001, 0.01, 0.1, 1, 10]}

grid = GridSearchCV(LogisticRegression(), param_grid, cv=5)
grid.fit(X_train, y_train)
print("Best cross-validation score: {}".format(grid.best_score_))
print("Best parameters: ", grid.best_params_)
#doing scoring on test data
print("{}".format(grid.score(X_test, y_test)))
grid_log_f1 = f1_score(y_test, log_y_pred)
print("Grid Logistic Reg - F1 score: {}".format(grid_log_f1))
#creating confusion matrix
#changed from y_pred to log_y_pred
cnf_matrix = confusion_matrix(y_test, log_y_pred)
cnf_matrix
#TF-IDF- Term frequency and inverse document frequency
from sklearn.pipeline import Pipeline, make_pipeline
pipeline = make_pipeline(TfidfVectorizer(min_df=5, norm=None),
                     LogisticRegression())
tfidf_param_grid = {'logisticregression__C': [0.001, 0.01, 0.1, 1, 10]}
tfidf_grid = GridSearchCV(pipeline, tfidf_param_grid, cv=5)
tfidf_grid.fit(x_train, y_train)

print("Best cross-validation score: {}".format(tfidf_grid.best_score_))
print("Best parameters: ", tfidf_grid.best_params_)
#Calculating accuracy on tf-idf using Logistic Regression
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
# Vectorize X_train
vectorizer_tf = CountVectorizer(min_df=5,ngram_range=(2, 2)).fit(x_train)
X_train_tf = vectorizer.transform(x_train)
print("X_train:\n{}".format(repr(X_train_tf)))
print("Vocabulary size: {}".format(len(vectorizer_tf.vocabulary_)))
feature_names_tf = vectorizer_tf.get_feature_names()
print("Number of features: {}".format(len(feature_names_tf)))
scores_tf = cross_val_score(LogisticRegression(), X_train_tf, y_train, cv=5)

print("CV accuracy: {}".format(np.mean(scores_tf)))
logreg_tf = LogisticRegression(C=0.1).fit(X_train_tf, y_train)
X_test_tf = vectorizer_tf.transform(x_test)
log_y_pred_tf = logreg_tf.predict(X_test_tf)
logreg_score_tf = accuracy_score(y_test, log_y_pred_tf)
print("Accuracy:   {}".format(logreg_score_tf))
#MNB on CountVectorizer
# Applying laplace smoothing parameter as 0.01

mnb = MultinomialNB(alpha=.01)
mnb.fit(X_train, y_train)
mnb_y_pred = mnb.predict(X_test)
mnb_score = accuracy_score(y_test, mnb_y_pred)
print("Accuracy:   {}".format(mnb_score))
print("Training set score: {}".format(mnb.score(X_train, y_train)))
print("Test set score: {}".format(mnb.score(X_test, y_test)))
mnb_cfm = confusion_matrix(y_test, mnb_y_pred)
print("Confusion matrix:")
print(mnb_cfm, end='\n\n')
print('-'*15)
print(np.array([['TN', 'FP'],[ 'FN' , 'TP']]))
plt.imshow(mnb_cfm, interpolation='nearest')

for i, j in itertools.product(range(mnb_cfm.shape[0]), range(mnb_cfm.shape[1])):
    plt.text(j, i, mnb_cfm[i, j],
             horizontalalignment="center",
             color="brown")

plt.ylabel('True label (Recall)')
plt.xlabel('Predicted label (Precision)')
plt.title('Multinomial | Confusion Matrix')
plt.colorbar();
mnb_f1 = f1_score(y_test, mnb_y_pred)
print("Multinomial NB - F1 score: {}".format(mnb_f1))
#TfidfVectorizer
# Vectorize X_train
mnb_tfidfvectorizer = TfidfVectorizer(min_df=5).fit(x_train)
mnb_X_train = mnb_tfidfvectorizer.transform(x_train)
print("X_train:\n{}".format(repr(mnb_X_train)))
mnb_X_test = mnb_tfidfvectorizer.transform(x_test)
mnb_score2 = accuracy_score(y_test, mnb_y_pred)
print("Accuracy:   {}".format(mnb_score2))
mnb_cfm2 = confusion_matrix(y_test, mnb_y_pred)
print("Confusion matrix:")
print(mnb_cfm2, end='\n\n')
print('-'*15)
print(np.array([['TN', 'FP'],[ 'FN' , 'TP']]))
plt.imshow(mnb_cfm2, interpolation='nearest')

for i, j in itertools.product(range(mnb_cfm2.shape[0]), range(mnb_cfm2.shape[1])):
    plt.text(j, i, mnb_cfm2[i, j],
             horizontalalignment="center",
             color="white")

plt.ylabel('True label (Recall)')
plt.xlabel('Predicted label (Precision)')
plt.title('Multinomial | Confusion Matrix')
plt.colorbar();
mnb2_f1 = f1_score(y_test, mnb_y_pred)
print("Multinomial NB - F1 score: {:.3f}".format(mnb2_f1))<jupyter_output>Multinomial NB - F1 score: 0.938<jupyter_text>Summary: Logistic Regression  gives the best result  Accuracy:   0.9018559762435041<jupyter_code># s="Overall, I'd say that this is a fantastic knife. The other glowing reviews for the knife are absolutely correct in terms of its solid feel and sharpness. The lock is excellent, and the serrated edge is just about perfect. For the price, I don't know if you can do any better in terms of folding knives.But then the unexpected let down.I took it with me on a kayaking trip to an island off the coast of Florida. I had three knives with me, all of them Gerber knives of different types. All three were stainless steel blades, one being a PFD knife, and the other a full tang non-folding knife.Over the course of the trip out there, all three knives were exposed to significant amounts of salt water. Both of the other knives had made this trip before, so I didn't give it a second thought.  I guess I should have, because opening the knife the next day revealed a blade covered in rust spots. Of course, I cleaned it off as best I could when I brought it home, but there are still a couple dozen rust marks on the face of the blade that just won't come off.So I know that it's my fault for not washing off the blade immediately. I guess since the other two knives have had no problems, I just didn't think about it. With those knives I can just wash the dried salt residue off of them whenever I get around to it. And with a limited fresh water supply (whatever I can fit in my kayak along with the rest of the gear), I need to be careful with how much I use.I guess to me the question is if Gerber can make these two other knives impervious to salt water, then why not this one?Maybe there are others out there with better experiences in salty environments, and if anyone would like to mention those experiences, feel free to leave a comment with this review. I welcome dissenting opinions, but I can only review based on my own personal experiences. And overall I love the knife, but it had to lose points for rusting after one day."
# s1=["This came in on time a​am veru happy with","hi there","Hello is there anybody in there","bad words","hate shit fuck"]
# def my_predict(s, vect. mod):
#     tok1=pd.Series(s)
#     x_test_observe=vect.transform(tok1)
#     return mod.predict(x_test_observe)
# my_predict=lambda s, vect, mod: mod.predict(vect.transform(pd.Series(s)))
# my_predict(s,vectorizer, logreg)
# my_predict=lambda s, vect, mod: mod.predict(vect.transform(pd.Series(s1)))
# my_predict(s1,vectorizer, logreg)
s1=["This came in on time a​am veru happy with","hi there","Hello is there anybody in there","bad words","hate shit fuck"]
my_predict=lambda s, vect, mod: mod.predict(vect.transform(pd.Series(s1)))
my_predict(s1,vectorizer, logreg)
x_train[:10]
y_train[:10]<jupyter_output><empty_output><jupyter_text>Reading 100 cell phone reviews for scoring to see how our model performs<jupyter_code>a= ["They look good and stick good! I just don't like the rounded shape because I was always bumping it and Siri kept popping up and it was irritating. I just won't buy a product like this again",
 "These stickers work like the review says they do. They stick on great and they stay on the phone. They are super stylish and I can share them with my sister. ",
 "These are awesome and make my phone look so stylish! I have only used one so far and have had it on for almost a year! CAN YOU BELIEVE THAT! ONE YEAR!! Great quality!",
 "Item arrived in great time and was in perfect condition. However",
 "awesome! stays on",
 "These make using the home button easy. My daughter and I both like them.  I would purchase them again. Well worth the price.",
 "Came just as described.. It doesn't come unstuck and its cute! People ask where I got them from & it's great when driving.",
 "it worked for the first week then it only charge my phone to 20%. it is a waste of money.",
 "Good case",
 "This is a fantastic case. Very stylish and protects my phone. Easy access to all buttons and features",
 "this case fits perfectly on the s4 and keeps me powerd all day I can't complain! a+ recommend it to all",
 "This is the first battery case I have had for my Galaxy S4. The S4 fits very well",
 "Performs exactly as advertised . It's very  sturdily built",
 "Unlike Most of the Rechargeable Battery cases",
 "Just what I needed. I needed a phone case for myself and my two sons",
 "When there is no outlets",
 "It works great. Doesn't heat up like crazy like the other ones I got",
 "Surprisingly",
 "I have tested this against the griffin dual output unit.I checked the charging current.This unit was charging my galaxy note battery with 70 ma.Griffin was charging with 40 ma! And the griffin was 4 times more expensive.I have not used these for very long. I bought 15 of them",
 "It worked great for the first couple of weeks then it just stopped completely.. so basically a small waste of money.",
 "I love that it has two ports for my phone and ipod. Who wants to be putting too many things in one socket.  Sleek and convenient to store and I just love it.",
 "just what you need",
 "does not have the need amps to charge things like ipads",
 "They are nothing special for sure",
 "I have several chargers. Have more than one vehicle so I keep more than one of these in each. Nothing more frustrating than finding all of a sudden one quits working",
 "I bought this a little skeptical. After I tried it I bought two more. It works great and so far it has lasted for about 3 months. If that changes I will update this review.",
 "I am disappointed that the 1A didn't work with my iPad.  That's what I get for buying a cheap adapter.",
 "This is a nice charger but you can tell it was made cheaply in China.  When it is charging the phone",
 "After a week only one side works",
 "Yo get exactly what you order in a timely fashion. And the item is just as described. Great buy if you ask me",
 "This is a terribly awesome product in my Subaru's console.  Passengers love it",
 "I've bough a munch of different things like this over the years. Most wouldn't stay in the jack",
 "Only works one side at a time. When you connect two cables",
 "it works great i like i can charge two thing at the same now i dont have to wait for my to finish charging her phone",
 "It came at last",
 "Didn't last very long. Worked great when it worked but it is a cheap piece of plastic crap so I shouldn't have expected it to last.",
 "excellent product",
 "Purchased two and we put one in each car. Now we charge our ipad and iohone together. Can also use to power my Galaxy Note 2 or recharge other USB devices. If you have to think about it you need counseling",
 "great charger for 2 devices and i haven't had any problems so far 3yrs now",
 "Purchased product about many a month ago. Pros",
 "I bought this so that I could use and charge my Tab at the same time. My tab does not recognize the high power port and will either charge while off or use power while on",
 "I bought this to charge my iphone and tablet in the car. works for iphone but not the tablet. I needed a stronger charge for that but this is still great for both iphoe and ipod together. it's cheap",
 "Works great. The blue led light is a nice touch in the car. We charge a Samsung and Iphone using this outlet. Nice low profile too.",
 "works great and charges ipads",
 "I could only give this USB car charger 2 stars because although it worked fine for about 3 months",
 "This charger works great and is short",
 "I bought 2 of this and tried to test first ... after few minutes of charging",
 "This portable USB Port car charger is a must have for people on the go.  Pair this with an extra or spare cord and you are good to go!",
 "I love the way it can charge 2 devices at once!  Seems to charge quicker without GPS on",
 "Bought so we could charge to phones at the same time while in the car from the same port.  Works.",
 "I received this product before I expected. It looks pretty good and It works with my Iphone (3GS) and my phone (HTC Evo V 3D). It is a good deal because It is not easy to find something like this for this price",
 "I have several of these",
 "good product at low price.purchased this looking for a smaller charger and I love Griffin products.Free shipping just took a little longer",
 "I use this in my car to charge my phone and my iPod as needed. It works well",
 "I love this case! It's so pretty.And I love the way the case feels to the touch because of the rubber. Very happy!",
 "The idea of the design is a sweet idea but it wears off as the paint is over the case and not under a sealant or anything. It has a cool 3D effect but at the cost of the paint rubbing off.It is quite pretty though.",
 "I was worried about this order because the picture on the description page kept changing to a less-desireable green/orange cover.  But alas",
 "Another pretty phone case that I really love. I have many as I love t o change them around .thanks",
 "The case is not white",
 "Defintiely worth the price...You should order several.  Most cases are $15 and up especially at the cell phone stores.  I can't believe the mark up on these things!",
 "This is a great case... It fit my phone perfectly and I still use it from time to time. Would recommend",
 "Received this item very quickly. The design is even more vivid than expected. The cover is soft (rubberized) but durable. I have received many compliments. It was an excellent buy! I would recommend this to anyone wanting a \"good look\" for their phone.",
 "this cover makes an old phone look and feel new. I like that I can order covers for little money and snazzy up my phone.",
 "I loved the case when I first received it but shortly after",
 "The color is more dark in person",
 "Simple to install and a great way to help protect your phone.My wife really liked the ability to change the look of her phone.",
 "The case is pretty good. It dose have some scratches on one side of the phone but its every little. Other then that i like it. Ive dropped my phone a few times since putting on the case and it didnt break or scratch. Ive gotten many complements on this case and over all it was worth the few bucks I spent.",
 "This case is afordable and yes it peels very easy but thats to be expected because its a dollor. I bought mine from Electromaster. It snaps on fine for me but if I were to drop my phone the phone would be fine but the case would surely be done for. its only a dollar so do not get your hopes up. it looks great but its for show not for protection. if you want your phone to be protected go the extra mile for an outter box",
 "This is one of my favorite cases I have this one on my phone I really love the color a lot.",
 "It looked like it was used! It was broken when it got to be the paint job was horrible.I would never get it again",
 "This case for some reason is peeling",
 "Its rubberized doesn't look like it in the picture but it is It's really cute I love this case used it for awhile but I switch out cases like I do my shoes.",
 "Received this in about a weeks time. Very nice actually considering how cheap the price was. It's a pink/purple with black leopard spots and has a very nice feel to it. I haven't tried to remove it yet",
 "This case was by far the best case I bought for this phone.  It is very stylish and I received many compliments",
 "This is the cutest case EVER!  I love it because it was easy to apply.  I had to switch from the rubber one because after so many uses",
 "my daughter liked it for a few days and then i didnt see it anymore on the phone. Use your judgement on that one.",
 "very good charger",
 "This product arrived when promised and in the condition promised. It is a genuine Blackberry charger. It was brand new and worked perfectly. For the price",
 "Great product. Use this with my Galaxy S4. At this price I bought 2 more. One for work and one to keep in the car incase I ever travel and forget my charger.",
 "Honestly I have always loved this specific model",
 "I've had this charger well over a year now",
 "so far no problems",
 "Great ! So far.I had this charger before and it seem to last pretty good 3 years fits most phones except Iphone and Galaxy 5  but that phone can still use it in the right side of the portal it will Judy charge it a bit slower. When this charger goes it tends to loose it's fit firmly in usb and has a short.",
 "Another very good phone charger that does the job and has a  very affordable price that anyone can afford  to buy it.",
 "Works well. We use this for travel and it lives in our luggage most of the year. For the low price having an extra USB charger this works great. Charges our Galaxy S3 and Note 2.",
 "I needed a charger quick and I didn't want to spend $20.00 at the convenience store for a short",
 "Bought as a spare charger for the house and turned into my main charger because my other one failed. Nice long cable and has held up for tons of charges for my Galaxy S3.",
 "The charger only lasted a week ad after that it just stopped charging my Samsung Galaxy S3.  They really need to start making these chargers better....I've been thru several!!!!",
 "This Charger is awesome! First of all I wasn't expecting it until between 12/12  and 12/17",
 "I purchase this for a backup at work.  It charges my SGS4.  I would recommend this charger for the Samsung Galaxy S4.",
 "This charger woks wonderfully on the Samsung Galaxy S4.  it chargers the phone fast and the price is great!  Highly Recommended product!",
 "This is another great product and i will always order from here in the future. money spent well!! If you need a great car charger this is the one you need!!",
 "goes into the lighter port a little rough but it charges the phone fine. I wish it had a little more reach so i could use the phone from the passenger seat of the car more comfortably but its not a big problem. it will reach pretty far because of the coiled section but this will put stress on the charge port of your phone which many people dont realize will eventually break it.",
 "I purchased two of these chargers for my sister and girlfriend. They both have Samsung Galaxy phones and needed a car charger so being the OEM fanboy I am",
 "I bought this for my Samsung Charge. It works perfectly and is built as solidly as the $20 Verizon charger. I highly recommend this car charger and thanks for the great price!",
 "It does what it needs to do",
 "Charger works great. Haven't had any issues with it at all. I wish it has a retractable cord like some other chargers I've used",
 "It works great  thus  far no issues.  The only thing i might have limited was a usb Connect or it charges my t989 without fail",
 "The big store that is in every town now (can't say their name) either does not carry anything or is always out of stock. If they ever have anything is stock",
 "Good lengthEasy to useVery fast chargingI would recommend it to persons who use their phone a lot."]

len(a)
my_predict=lambda a, vect, mod: mod.predict(vect.transform(pd.Series(a)))
my_predict(a,vectorizer, logreg)
df.head()
df_=df[["reviewText","pos_neg"]]
df_['vader_polarity']=0.0
df_.head()
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyser = SentimentIntensityAnalyzer()
def sentiment_analyzer_scores(sentence):
    score = analyser.polarity_scores(sentence)
#     print("{} {}".format(sentence, str(score)))
    if score['compound'] >= 0.05:
        return 1 
    else:
        return 0
df_temp=df_[:100]
df_len=len(df_temp)
df_len
for index,row in df_temp.iterrows():
    print("{} remaining to process".format(df_len), end='\r')
    df_temp.iloc[index, df_temp.columns.get_loc('vader_polarity')] = float(sentiment_analyzer_scores(row['reviewText']))
    df_len-=1
df_temp.dtypes
df_temp.head(10)
df_temp['diff'] = np.where(df_temp['vader_polarity']==df_temp['pos_neg'],'Equal', "Unequal")
for items in df_temp.loc[df_temp['diff'] == 'Unequal'].iterrows():
    print ("\nText: {} \nvader polarity: {} \npos_neg: {}" .format(items[1]['reviewText'], items[1]['vader_polarity'],
                                                              items[1]['pos_neg']))
vader_pol=[]
for sen in a:
    vader_pol.append(sentiment_analyzer_scores(sen))
print (vader_pol)
mod_predict=my_predict(a,vectorizer, logreg)
final_pred=[]
for i in range(0, len(vader_pol)):
    if vader_pol[i]==mod_predict[i]:
        final_pred.append(vader_pol[i])
    else:
        final_pred.append(0)
for i in range(0, len(a)):
    print ("\nText: {} \nfinal polarity: {}" .format(a[i], final_pred[i]))<jupyter_output>Text: They look good and stick good! I just don't like the rounded shape because I was always bumping it and Siri kept popping up and it was irritating. I just won't buy a product like this again 
final polarity: 0

Text: These stickers work like the review says they do. They stick on great and they stay on the phone. They are super stylish and I can share them with my sister.  
final polarity: 1

Text: These are awesome and make my phone look so stylish! I have only used one so far and have had it on for almost a year! CAN YOU BELIEVE THAT! ONE YEAR!! Great quality! 
final polarity: 1

Text: Item arrived in great time and was in perfect condition. However 
final polarity: 1

Text: awesome! stays on 
final polarity: 1

Text: These make using the home button easy. My daughter and I both like them.  I would purchase them again. Well worth the price. 
final polarity: 1

Text: Came just as described.. It doesn't come unstuck and its cute! People ask where I got them from & it's great when[...]
{'fraction_non_alphanumeric': 0.08432299320808674, 'fraction_numerical': 0.009214759502720738, 'mean_word_length': 4.47705025016315, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 3, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '466967', 'license': 'Apache-2.0', 'n_tokens_mistral': 7695, 'n_tokens_neox': 7386, 'n_words': 3632, 'path': 'com/src/ecomm_reviews.ipynb', 'repo_name': 'hardik-goel/Sentiment-Analysis', 'size': '25182'}