<jupyter_start><jupyter_text>Sentiment Analysis: Sentiment LexiconLa idea es incorporar información externa acerca de la presencia de palabras positivas y negativas.Opciones:1. sustituir las palabras por marcadores especiales POS y NEG.2. agregar nuevos features numéricos que indiquen la presencia/cantidad de POS y NEG.Vamos por la 2.Usamos:- https://mpqa.cs.pitt.edu/lexicons/subj_lexicon/<jupyter_code>%load_ext autoreload
%autoreload 2
from util import load_datasets
train, dev, test = load_datasets()
X_train, y_train = train
X_dev, y_dev = dev
X_test, y_test = test<jupyter_output><empty_output><jupyter_text>Estado del Arte Actual<jupyter_code>from model import build_pipeline
from util import print_eval

pipeline = build_pipeline()
pipeline.fit(X_train, y_train)
print_eval(pipeline, X_dev, y_dev)<jupyter_output>accuracy	0.89

             precision    recall  f1-score   support

        neg       0.87      0.93      0.90       162
        pos       0.91      0.84      0.87       138

avg / total       0.89      0.89      0.89       300

[[150  12]
 [ 22 116]]<jupyter_text>Carga del Lexicón<jupyter_code>filename = 'subjectivity_clues_hltemnlp05/subjclueslen1-HLTEMNLP05.tff'
f = open(filename)
lines = f.readlines()
f.close()
words = []
for line in lines:
    sline = line.split()
    dline = dict([token.split('=') for token in sline if '=' in token])
    word = dline['word1']
    pol = dline['priorpolarity']
    if pol not in {'both', 'neutral'}:
        if pol in {'negative', 'weakneg'}:
            pol = 'NEG'
        else:
            pol = 'POS'
        words.append((word, pol))

word_dict = dict(words)
word_dict['abandon']<jupyter_output><empty_output><jupyter_text>Nuevo Tokenizer<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
tkn = CountVectorizer().build_tokenizer()
def my_tkn(s):
    tokens = tkn(s)
    return [word_dict.get(token, token) for token in tokens]
my_tkn('creaky bastard')
#print(X_dev[0].decode('utf-8')[:200])
#my_tkn(X_dev[0].decode('utf-8'))
vect = CountVectorizer(vocabulary=['POS', 'NEG'], tokenizer=my_tkn)
vect.fit(X_train)
vect.transform(X_train[:10]).toarray()
#vect.get_feature_names()<jupyter_output><empty_output><jupyter_text>Scaler<jupyter_code>from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
vect = Pipeline([
    ('pol', CountVectorizer(vocabulary=['POS', 'NEG'], tokenizer=my_tkn)),
    ('scl', StandardScaler(with_mean=False)),
])
vect.fit(X_train)
vect.transform(X_train[:10]).toarray()<jupyter_output>/home/francolq/.virtualenvs/pln/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)<jupyter_text>Feature Union<jupyter_code>from sklearn.pipeline import FeatureUnion
vect = FeatureUnion([
    ('bow', CountVectorizer(binary=True)),
    ('pol', 
         Pipeline([
            ('pol', CountVectorizer(vocabulary=['POS', 'NEG'], tokenizer=my_tkn)),
            ('scl', StandardScaler(with_mean=False)),
        ])
    )
])
vect.fit(X_train)
vect.transform(X_train[:1])<jupyter_output>/home/francolq/.virtualenvs/pln/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)<jupyter_text>Experimentos<jupyter_code>from sklearn.pipeline import FeatureUnion
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from util import print_eval

pipeline = Pipeline([
    ('vect', FeatureUnion([
        ('bow', CountVectorizer(binary=True)),
        ('pol', 
             Pipeline([
                ('pol', CountVectorizer(vocabulary=['POS', 'NEG'], tokenizer=my_tkn)),
                ('scl', StandardScaler(with_mean=False)),
            ])
        )
    ])),
    ('clf', LogisticRegression(random_state=0)),
])
pipeline.fit(X_train, y_train)
print_eval(pipeline, X_dev, y_dev)
from sklearn.pipeline import FeatureUnion
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from util import print_eval

pipeline = Pipeline([
    ('vect', FeatureUnion([
        ('bow', CountVectorizer(
            binary=True,
            min_df=3,
            max_df=0.90,
            ngram_range=(1, 5),
        )),
        ('pol', 
             Pipeline([
                ('pol', CountVectorizer(vocabulary=['POS', 'NEG'], tokenizer=my_tkn)),
                ('scl', StandardScaler(with_mean=False)),
            ])
        )
    ])),
    ('clf', LogisticRegression(random_state=0)),
])
pipeline.fit(X_train, y_train)
print_eval(pipeline, X_dev, y_dev)<jupyter_output>/home/francolq/.virtualenvs/pln/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
{'fraction_non_alphanumeric': 0.12141882673942701, 'fraction_numerical': 0.020853634769050867, 'mean_word_length': 4.111553784860558, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '86547', 'license': 'MIT', 'n_tokens_mistral': 1801, 'n_tokens_neox': 1751, 'n_words': 347, 'path': 'archive/2018/sentiment/demo8.ipynb', 'repo_name': 'guillepaez53/AprendizajeSupervisado', 'size': '5131'}