<jupyter_start><jupyter_text>TODO: Figure out ravel() 1d array problem<jupyter_code>import pandas as pd
import numpy as np
import os
from nltk import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
from __future__ import division
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
from sklearn.feature_extraction.text import TfidfVectorizer
labels = pd.read_csv('stocknews//labels.csv', header=None)<jupyter_output><empty_output><jupyter_text>http://stackoverflow.com/questions/28382735/python-pandas-does-not-read-the-first-row-of-csv-file`pd.read_csv` was cutting off the first row of labels<jupyter_code># Confirm size of labels to make sure data loaded correctly
labels.shape
str_tokens_all = pd.read_csv('stocknews//tokens_str_all.csv', dtype=str, keep_default_na=False)
str_tokens_all.head(2)
str_tokens_all.shape
def replace_num(element):
    return ' '.join([('numero' if k.isdigit() else k) for k in element.split()])<jupyter_output><empty_output><jupyter_text>http://stackoverflow.com/questions/6905636/python-conditional-list-joins Instead of a digit, use `'NUMERO'` as it is resistant to stemming/lemmatizing.  It's possible that headlines with numbers might contain some information.  This will prevent preprocessing from discarding the information.  Tag parts-of-speech and lemmatize the text<jupyter_code>sample = str_tokens_all.iloc[0,0]
from nltk import pos_tag
from nltk.tokenize import word_tokenize
 
s = "This is a simple sentence"
tokens = word_tokenize(s) # Generate list of tokens
tokens_pos = pos_tag(tokens)
tokens_pos
type(tokens)
# Need to reconcile treebank and Wordnet tags
# http://stackoverflow.com/questions/15586721/wordnet-lemmatization-and-pos-tagging-in-python
from nltk.corpus import wordnet

def get_wordnet_pos(tag):
    if tag.startswith('V'):
        return wordnet.VERB
    elif tag.startswith('N'):
        return wordnet.NOUN
    elif tag.startswith('J'):
        return wordnet.ADJ
    elif tag.startswith('R'):
        return wordnet.ADV
    elif tag.startswith('S'):
        return wordnet.ADJ_SAT
    else:
        return wordnet.NOUN
        # needs something there or else key error
p_o_s = {
    'N' : wordnet.NOUN,
    'V' : wordnet.VERB,
    'J' : wordnet.ADJ,
    'S' : wordnet.ADJ_SAT,
    'R' : wordnet.ADV
}
part = {
    'N' : 'n',
    'V' : 'v',
    'J' : 'a',
    'S' : 's',
    'R' : 'r'
}
def convert_tag(penn_tag):
    '''
    convert_tag accepts the first letter ofa Penn part-of-speech tag
    from nltk.pos_tag() and then uses a dict to converts it to the 
    appropriate WordNet tag.
    '''
    if penn_tag in part.keys():
        return part[penn_tag]
    else:
        return 'n'
tokens_pos[1][1][0] in part.keys()
from nltk.stem import WordNetLemmatizer
wnl = WordNetLemmatizer()
# Simple function to tag part of speech (pos) to a sentence and return lemmatized word.
# However, cannnot do it all in one step, need to pos tag before splitting sentence up.

def tag_and_lem(element):
    sent = pos_tag(element.split()) # list of tuples [('token', 'Treebank tag')...]
    return ' '.join([wnl.lemmatize(sent[k][0], convert_tag(sent[k][1][0]))for k in range(len(sent))])
    #returns str
tag_and_lem(sample)<jupyter_output><empty_output><jupyter_text>Tag and lemmatize entire corpus This will take some time...<jupyter_code># Tag and lem but not merge
lemma_str_tokens_all = str_tokens_all.applymap(tag_and_lem)
lemma_str_tokens_all.head(2)<jupyter_output><empty_output><jupyter_text>Merge Cells<jupyter_code>lemma_str_tokens_all['merged'] = lemma_str_tokens_all.iloc[:, 0:].apply(lambda x: ' '.join(x.dropna().values.tolist()), axis=1)
lemma_str_tokens_all['merged'] = lemma_str_tokens_all['merged'].apply(replace_num)<jupyter_output><empty_output><jupyter_text>Split data into train and test sets<jupyter_code>train_text = lemma_str_tokens_all.merged[0:1493] # train features
test_text = lemma_str_tokens_all.merged[1493:] # test features
train_labels = labels[0:1493].values # train labels
test_labels = labels[1493:].values; # test labels
vectorizer = TfidfVectorizer( max_features=250000, ngram_range=(1, 1), sublinear_tf=True, stop_words='english')
# Only need text, not labels
train_x = vectorizer.fit_transform( train_text )
test_x = vectorizer.transform( test_text )
train_x.shape
test_x.shape<jupyter_output><empty_output><jupyter_text>Passive Aggressive Classifier<jupyter_code>from sklearn.linear_model import PassiveAggressiveClassifier
classifier = PassiveAggressiveClassifier(n_iter=25)
train_labels.shape
train_labels.ravel()
classifier.fit(train_x, train_labels.ravel())
classifier.score(test_x, test_labels.ravel())<jupyter_output><empty_output><jupyter_text>SGD Classifier<jupyter_code>from sklearn.linear_model import SGDClassifier
classifier = SGDClassifier(loss='squared_loss', n_iter=8)
classifier.fit(train_x, train_labels.ravel())
classifier.score(test_x, test_labels.ravel())<jupyter_output><empty_output><jupyter_text>Ridge Classifier<jupyter_code>from sklearn.linear_model import RidgeClassifier
clf = RidgeClassifier()
clf.fit(train_x, train_labels.ravel())
clf.score(test_x, test_labels.ravel())<jupyter_output><empty_output><jupyter_text>Gaussian Naive Bayes<jupyter_code>from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB()
gnb.fit(train_x.toarray(), train_labels.ravel())
gnb.score(test_x.toarray(), test_labels.ravel())<jupyter_output><empty_output><jupyter_text>Support Vector Classifier<jupyter_code>from sklearn.svm import SVC
supportvc = SVC()
supportvc.fit(train_x, train_labels.ravel())
supportvc.score(test_x, test_labels.ravel())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11167870354060243, 'fraction_numerical': 0.011449709353531795, 'mean_word_length': 5.890776699029126, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '597377', 'license': 'MIT', 'n_tokens_mistral': 2062, 'n_tokens_neox': 1976, 'n_words': 431, 'path': 'BOW_Lemmatized_Unigrams.ipynb', 'repo_name': 'KT12/Capstone_in_progress', 'size': '5677'}