<jupyter_start><jupyter_text>perform sentiment analysis<jupyter_code>import numpy as np
import pickle
import collections
import nltk.classify.util, nltk.metrics
from nltk.classify import NaiveBayesClassifier, MaxentClassifier, SklearnClassifier
import csv
from sklearn import cross_validation
from sklearn.svm import LinearSVC, SVC
import random
from nltk.corpus import stopwords
import itertools
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures
import pandas as pd<jupyter_output>C:\Users\ADITYA\Anaconda3\envs\ml\lib\site-packages\sklearn\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)<jupyter_text>unigrams features<jupyter_code>#Finding the unigram representation
from sklearn.feature_extraction.text import CountVectorizer
vectorizer=CountVectorizer()
with open('sentiments/SentimentText_train.txt', 'rb') as f:
    SentimentText = pickle.load(f)
X=vectorizer.fit_transform(SentimentText)
train_data = pd.read_csv('data/train.csv', encoding = 'ISO-8859-1')
y = train_data['Sentiment']
y.shape
X.shape
from sklearn.model_selection import train_test_split
X_train, X_test, y_train ,y_test = train_test_split(X,y,train_size = .8 , test_size = .2 , random_state = 0)
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB()
clf.fit(X, y)
# training score
clf.score(X_train,y_train)
# testing score
clf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>using bigrams features<jupyter_code># #Finding the bigram representation 
bigram_vectorizer=CountVectorizer(ngram_range=(1,2))
with open('sentiments/SentimentText_train.txt', 'rb') as f:
    SentimentText = pickle.load(f)
X=bigram_vectorizer.fit_transform(SentimentText)
train_data = pd.read_csv('data/train.csv', encoding = 'ISO-8859-1')
y = train_data['Sentiment']
X.shape
y.shape
from sklearn.model_selection import train_test_split
X_train, X_test, y_train ,y_test = train_test_split(X,y,train_size = .8 , test_size = .2 , random_state = 0)
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB()
clf.fit(X, y)
# training score
clf.score(X_train, y_train)
# testing score
clf.score(X_test, y_test)
with open('sentiments/SentimentText_test.txt', 'rb') as f:
    SentimentText = pickle.load(f)
X_test=bigram_vectorizer.transform(SentimentText)
X_test.shape
predicted = clf.predict(X_test)
predicted.shape
output = pd.read_csv('data/test.csv', encoding = 'ISO-8859-1')
output.drop(['SentimentText'],axis = 1)
output['Sentiment'] = predicted
output.to_csv('output.csv', index = False)<jupyter_output><empty_output><jupyter_text>using both unigrams and bigrams as features set<jupyter_code>#Finding the unigram representation
from sklearn.feature_extraction.text import CountVectorizer
vectorizer=CountVectorizer()

# #Finding the bigram representation 
bigram_vectorizer=CountVectorizer(ngram_range=(1,2))
with open('sentiments/SentimentText_train.txt', 'rb') as f:
    SentimentText = pickle.load(f)
X_unigrams = vectorizer.fit_transform(SentimentText)
X_bigrams = bigram_vectorizer.fit_transform(SentimentText)

X_unigrams.shape
X_bigrams.shape
from scipy.sparse import hstack 
X_combined = hstack([X_unigrams , X_bigrams])
X_combined.shape
train_data = pd.read_csv('data/train.csv', encoding = 'ISO-8859-1')
y = train_data['Sentiment']
clf = MultinomialNB()
clf.fit(X_combined, y)
with open('sentiments/SentimentText_test.txt', 'rb') as f:
    SentimentText = pickle.load(f)
X_test_unigrams = vectorizer.transform(SentimentText)
X_test_bigrams = bigram_vectorizer.transform(SentimentText)
X_test_unigrams.shape
X_test_bigrams.shape
from scipy.sparse import hstack 
X_test_combined = hstack([X_test_unigrams , X_test_bigrams])
X_test_combined.shape
predicted = clf.predict(X_test_combined)
output = pd.read_csv('data/test.csv', encoding = 'ISO-8859-1')
output.drop(['SentimentText'],axis = 1)
output['Sentiment'] = predicted
output.to_csv('output.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1020121665886757, 'fraction_numerical': 0.011464670098268601, 'mean_word_length': 7.268858800773694, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '506959', 'license': 'MIT', 'n_tokens_mistral': 1550, 'n_tokens_neox': 1479, 'n_words': 321, 'path': '.ipynb_checkpoints/program-checkpoint.ipynb', 'repo_name': 'adibyte95/Twittter-sentiment-analysis', 'size': '4274'}