<jupyter_start><jupyter_text><jupyter_code>import pandas as pd

df = pd.read_csv('movie_data.csv')

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
count = CountVectorizer()

docs = np.array(['The sun is shining',
                'The weather is sweet',
                'The sun is shining, The weather is sweet, and one and one is two'])

bag = count.fit_transform(docs)
print(count.vocabulary_)
# vocab is stored in a dictionary which maps the unique words in the document to integers
# now to print a feature vector we convert bag into an array
print(bag.toarray()) 
from sklearn.feature_extraction.text import TfidfTransformer
np.set_printoptions(precision=2)
tfidf = TfidfTransformer(use_idf = True, norm ='l2', smooth_idf = True)

print(tfidf.fit_transform(bag).toarray())
#data preparation
df.loc[0,'review'][-50:]
import re
def preprocessor(text):
  text = re.sub('<[^<.]*>','',text)
  emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(D|P)',text)
  text = re.sub('[\W]+',' ',text.lower()) +\
      ' '.join(emoticons).replace('-','')
  return text
preprocessor(df.loc[0,'review'][-50:])
df['review'] = df['review'].apply(preprocessor)<jupyter_output><empty_output><jupyter_text>Task 5:<jupyter_code>from nltk.stem.porter import PorterStemmer

porter = PorterStemmer()
def tokenizer(text):
  return text.split()
def tokenizer_stemming(text):
  return [porter.stem(word) for word in text.split()]
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
stop = stopwords.words('english')
[w for w in tokenizer_stemming('a runner likes running and runs a lot')[-10:]if w not in stop]<jupyter_output><empty_output><jupyter_text>Task 6: Transfor Text Data to TF-IDF Vectors<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(strip_accents=None,
                        lowercase = False,
                        preprocessor = None,
                        tokenizer = tokenizer_stemming,
                        use_idf = True,
                        norm = 'l2',
                        smooth_idf = True)
y = df.sentiment.values
X = tfidf.fit_transform(df.review)<jupyter_output><empty_output><jupyter_text>Task 7:Document Classification using Logistic Regression<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    random_state = 1,
                                                    test_size = 0.5,
                                                    shuffle = False)

import pickle
from sklearn.linear_model import LogisticRegressionCV

clf = LogisticRegressionCV(cv = 5,
                           scoring = 'accuracy',
                           random_state = 0,
                           n_jobs = -1,
                           verbose = 3,
                           max_iter = 300).fit(X_train, y_train)

saved_model = open('saved_model.sav', 'wb')
pickle.dump(clf,saved_model)
saved_model.close()<jupyter_output>[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.
[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  2.5min finished<jupyter_text>Task 8: Model Evaluation<jupyter_code>filename = 'saved_model.sav'
saved_clf = pickle.load(open(filename, 'rb'))
saved_clf.score(X_test, y_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11587348507242093, 'fraction_numerical': 0.009459060005911913, 'mean_word_length': 2.981176470588235, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '190281', 'license': 'MIT', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1062, 'n_words': 245, 'path': 'sentiment_analysis_with_scikit_learn.ipynb', 'repo_name': 'kratikagupta6394/sentiment-analysis-with-scikit-learn', 'size': '3383'}