<jupyter_start><jupyter_text>Import modules<jupyter_code>import pandas as pd
from sklearn.cross_validation import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics<jupyter_output><empty_output><jupyter_text>Read the dataset<jupyter_code>data = pd.read_table('tweets.tsv', usecols=[0,1])<jupyter_output><empty_output><jupyter_text>Split the data into features and labels<jupyter_code>X = data.SentimentText
y = data.Sentiment<jupyter_output><empty_output><jupyter_text>split data into train and test sets<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)<jupyter_output><empty_output><jupyter_text>Instantiate the `Vectorizer`<jupyter_code>vectorizer = CountVectorizer()<jupyter_output><empty_output><jupyter_text>Fit training data to the vectorizer<jupyter_code>vectorizer.fit(X_train)<jupyter_output><empty_output><jupyter_text>Create the Document Term Matrix<jupyter_code>X_train_dtm = vectorizer.transform(X_train)
X_test_dtm = vectorizer.transform(X_test)<jupyter_output><empty_output><jupyter_text>Build and Train a model<jupyter_code>nb = MultinomialNB()
nb.fit(X_train_dtm, y_train)
y_pred_class = nb.predict(X_test_dtm)
metrics.accuracy_score(y_test, y_pred_class)<jupyter_output><empty_output><jupyter_text>Tuning the Count Vectorizer  Default<jupyter_code>vectorizer
vectorizer = CountVectorizer(stop_words='english')
vectorizer = CountVectorizer(ngram_range=(1, 2))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09520725388601037, 'fraction_numerical': 0.003238341968911917, 'mean_word_length': 13.439252336448599, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '454094', 'license': 'MIT', 'n_tokens_mistral': 577, 'n_tokens_neox': 565, 'n_words': 81, 'path': 'Code/Section 4/Build and train a model.ipynb', 'repo_name': 'PacktPublishing/Pandas-for-Predictive-Analysis-using-scikit-learn-', 'size': '1544'}