import numpy as np
import re
from nltk.corpus import stopwords
import nltk
import logging
from gensim.models import word2vec


def get_dataset():
    files = ['./analysis/input/negative_tweets.txt', './analysis/input/neutral_tweets.txt', './analysis/input/positive_tweets.txt']

    x = []
    for file in files:
        s = []
        with open(file, 'r') as f:
            for line in f:
                s.append(line.strip())

        assert len(s) == 1367
        x.extend(s)

    y = np.array([-1] * 1367 + [0] * 1367 + [1] * 1367)
    return x, y


def sentence_to_wordlist(sentence, remove_stopwords=False):
    review_text = re.sub('[^a-zA-Z]', ' ', sentence)
    words = review_text.lower().split()
    if remove_stopwords:
        stops = set(stopwords.words("english"))
        words = [w for w in words if not w in stops]

    return words


def tweet_to_sentences(review, tokenizer, remove_stopwords=False):
    raw_sentences = tokenizer.tokenize(review.strip())
    sentences = []
    for raw_sentence in raw_sentences:
        if len(raw_sentence) > 0:
            sentences.append(sentence_to_wordlist(raw_sentence, remove_stopwords))

    return sentences


punkt_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
X, Y = get_dataset()
sentences = []

print('Parsing sentences from training set')
for tweet in X:
    sentences += tweet_to_sentences(tweet, punkt_tokenizer)

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
num_features = 300  # Word vector dimensionality
min_word_count = 10  # Minimum word count
num_workers = 4  # Number of threads to run in parallel
context = 10  # Context window size
downsampling = 1e-3  # Downsample setting for frequent words

print('Training model...')
model = word2vec.Word2Vec(sentences, workers=num_workers, size=num_features, min_count=min_word_count, window=context, sample=downsampling)

# If you don't plan to train the model any further, calling
# init_sims will make the model much more memory-efficient.
model.init_sims(replace=True)

# It can be helpful to create a meaningful model name and
# save the model for later use. You can load it later using Word2Vec.load()
model_name = '300features_40minwords_10context'
model.save(model_name)

import code; code.interact(local=dict(globals(), **locals()))

Add simple word2vec to train model