<jupyter_start><jupyter_text>Import modules, load data, split<jupyter_code>import csv
import json
import pandas as pd
import numpy as np
import re

import nltk
from nltk import Text
from nltk.tokenize import regexp_tokenize
from nltk.tokenize import word_tokenize  
from nltk.tokenize import sent_tokenize 
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression 
from sklearn.naive_bayes import MultinomialNB, GaussianNB

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline<jupyter_output><empty_output><jupyter_text>Importing tweets (the original data file)<jupyter_code>#Pprint is 'pretty print', simply a print function that gives 'nicer' outputs than print
from pprint import pprint

file_test = 'geotagged_tweets_20160812-0912.jsons'
tweets_file = open(file_test, "r")
tweets_data = []

for line in tweets_file:
    try:
        tweet = json.loads(line)
        tweets_data.append(tweet)
    except Exception as e:
        print (e)
        continue<jupyter_output><empty_output><jupyter_text>Build pandas dataframe with text, language & country<jupyter_code>tweets_test = pd.DataFrame()

tweets_test['text'] =    list(map(lambda tweet: tweet['text'], tweets_data))
tweets_test['lang'] =    list(map(lambda tweet: tweet['lang'], tweets_data))
tweets_test['type'] = list(map(lambda tweet: tweet['place']['place_type'] if tweet['place'] != None else None, tweets_data))
tweets_test['location'] = list(map(lambda tweet: tweet['place']['full_name'] if tweet['place'] != None else None, tweets_data))
tweets_test['country'] = list(map(lambda tweet: tweet['place']['country'] if tweet['place'] != None else None, tweets_data))
# tweets_test['text']
tweets_test.head()
tweets_test.iloc[12]<jupyter_output><empty_output><jupyter_text>Read the training file from csv<jupyter_code>file_train = 'kaggletweets.csv'
# file_test = 'tweetdata.csv'
cols = ['target', 'ids', 'date', 'flag', 'user', 'text']

tweets_train = pd.read_csv(file_train, delimiter=',', encoding='latin1', header=None, names=cols)
# tweets_test = pd.read_csv(file_test, delimiter=',', encoding='latin1')<jupyter_output><empty_output><jupyter_text>Clean tweets from hyperlinks and mentions<jupyter_code># training set
tweets_train.text = tweets_train.text.apply(lambda tweet: re.sub(r'https\S+', '', tweet))
tweets_train.text = tweets_train.text.apply(lambda tweet: re.sub(r'http\S+', '', tweet))
tweets_train.text = tweets_train.text.apply(lambda tweet: re.sub(r'@\S+', '', tweet))

# test set
tweets_test.text = tweets_test.text.apply(lambda tweet: re.sub(r'https\S+', '', tweet))
tweets_test.text = tweets_test.text.apply(lambda tweet: re.sub(r'http\S+', '', tweet))
# tweets_test.text = tweets_test.text.apply(lambda tweet: re.sub(r'@\S+', '', tweet))
# only want english tweets from us as the tfidfvectorizer will work only with english words
tweets_test = tweets_test[(tweets_test.lang == 'en') & (tweets_test.country == 'United States')]
tweets_test.head()<jupyter_output><empty_output><jupyter_text>Define training and test set with train_test_splitTo assess the quality of the model we split the tweets from the Kaggle into training and test sets as well. THIS IS A DIFFERENT TEST SET THAT IS ONLY USED TO CHECK MODEL PERFORMANCE. When the model is trained we use 'tweets_test' to predict labels.<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(tweets_train.text, tweets_train.target,
                                                   train_size=0.1, test_size=0.02, random_state=42)<jupyter_output><empty_output><jupyter_text>Tf-idf Vectorizer settings<jupyter_code># remove stop words only, no stemming or lemmatization
# punctuation is completely ignored
word_vectorizer = TfidfVectorizer(
    stop_words='english',
    lowercase=True,
    sublinear_tf=True,
    strip_accents='unicode',
    analyzer='word',
    token_pattern=r'\w{2,}',
    ngram_range=(1, 2))<jupyter_output><empty_output><jupyter_text>Fit vectorizer to the training set<jupyter_code>word_vectorizer.fit(X_train)
# X_train_word_features = word_vectorizer.transform(X_train)
# test_features = word_vectorizer.transform(X_test)<jupyter_output><empty_output><jupyter_text>Transform test set with fitted vectorizer<jupyter_code># not too sure if this is necessary as it wont work with predict
test_features = word_vectorizer.transform(tweets_test.text)<jupyter_output><empty_output><jupyter_text>Specify classifier and create pipeline<jupyter_code>classifier = MultinomialNB()
pipe = make_pipeline(word_vectorizer, classifier)
param_grid = {'multinomialnb__alpha': [0.1, 0.3, 0.5, 0.7, 0.9]}
grid = GridSearchCV(pipe, param_grid, cv=5)<jupyter_output><empty_output><jupyter_text>Fit model with grid search<jupyter_code>grid.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Predict labels for the test features<jupyter_code>sentiment_predictions = grid.predict(tweets_test.text)
# 0 = neg, 2 = neutral, 4 = pos
tweets_test['target'] = sentiment_predictions
tweets_test.head()<jupyter_output><empty_output><jupyter_text>Write results to csv<jupyter_code>outfile = 'tweet_classifications2.csv'
tweets_test.to_csv(outfile, sep=',', encoding='utf-8', index=False)
read_table = pd.read_csv('tweet_classifications2.csv')
read_table.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11099476439790576, 'fraction_numerical': 0.0075043630017452, 'mean_word_length': 6.915745856353591, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '155877', 'license': 'MIT', 'n_tokens_mistral': 2036, 'n_tokens_neox': 1939, 'n_words': 434, 'path': 'Twitter Sentiment Analysis.ipynb', 'repo_name': 'stijnuijen/Twitter-Sentiment-Analysis', 'size': '5730'}