<jupyter_start><jupyter_text>![image.png](attachment:image.png)  Stanford NLP API for opinoin mining[learn more](https://deepai.org/machine-learning-model/sentiment-analysis)<jupyter_code># sarcasm
import requests
r = requests.post(
    "https://api.deepai.org/api/sentiment-analysis",
    data={
        'text': 'i was killed by murder thanks your bonuses',
    },
    headers={'api-key': 'quickstart-QUdJIGlzIGNvbWluZy4uLi4K'}
)
print(r.json())<jupyter_output>{'status': "Looks like you're enjoying our API. Want to keep using it? Sign up to get an API Key that's as unique as you are. It's free! https://deepai.org/"}<jupyter_text>![image.png](attachment:image.png)  Problem----Train model for your own and compare with Stanford NLP. Pipline1) clean and vectorize dataset 2) train model 3) get scores (f1, eccuracy etc) 4) compare with same sentence<jupyter_code>import collections
import nltk
import os
from sklearn import (
    datasets, model_selection, feature_extraction, linear_model
)
def extract_features(corpus):
    '''Extract TF-IDF features from corpus'''
    # vectorize means we turn non-numerical data into an array of numbers
    count_vectorizer = feature_extraction.text.CountVectorizer(
        lowercase=True,  # for demonstration, True by default
        tokenizer=nltk.word_tokenize,  # use the NLTK tokenizer
        stop_words='english',  # remove stop words
        min_df=1  # minimum document frequency, i.e. the word must appear more than once.
    )
    processed_corpus = count_vectorizer.fit_transform(corpus)
    processed_corpus = feature_extraction.text.TfidfTransformer().fit_transform(
        processed_corpus)

    return processed_corpus
import pandas as pd
data = pd.read_csv('train_tweets.csv').reset_index(drop=True).drop('id', axis=1)
data.head(10)
data.info()
%matplotlib inline
data.label.hist()
tweet_tfidf = extract_features(data.tweet)
X_train, X_test, y_train, y_test = model_selection.train_test_split(
    tweet_tfidf, data.label, test_size=0.30, random_state=42)
print(X_train[:1])
# similar to nltk.NaiveBayesClassifier.train()
model = linear_model.LogisticRegression(class_weight='balanced', solver='sag')
model.fit(X_train, y_train)
print('Model performance: {}'.format(model.score(X_test, y_test)))
help(linear_model.LogisticRegression)
y_pred = model.predict(X_test)
from matplotlib.pyplot import hist
hist(y_pred)
hist(y_test)
from sklearn.metrics import f1_score
f1_score(y_test, y_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10972166196046793, 'fraction_numerical': 0.007664380798709157, 'mean_word_length': 5.2784810126582276, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '586994', 'license': 'MIT', 'n_tokens_mistral': 840, 'n_tokens_neox': 820, 'n_words': 200, 'path': 'week5/week 5 practice.ipynb', 'repo_name': 'sergeyfitts/hse_nlp_advanced', 'size': '2479'}