<jupyter_start><jupyter_text>Machine learning  Imports<jupyter_code>import sys
import cufflinks
import pandas as pd
import numpy as np
from tqdm import tqdm
import warnings

warnings.filterwarnings('ignore')

sys.path.append('./..')
cufflinks.go_offline()
from Corpus.Corpus import get_corpus, filter_binary_pn, filter_corpus_small
from auxiliar.VectorizerHelper import vectorizer, vectorizerIdf, preprocessor
from auxiliar import parameters
from auxiliar.HtmlParser import HtmlParser
from sklearn.calibration import CalibratedClassifierCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import roc_auc_score
from sklearn.metrics import f1_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score
from sklearn.metrics import roc_curve
from sklearn.model_selection import KFold
import copy<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>polarity_dim = 3
clasificadores=['lr', 'ls', 'mb', 'rf']
idf =  False
target_names=['Neg', 'Pos']
kfolds = 10
base_dir = '2-clases' if polarity_dim == 2 else ('3-clases' if polarity_dim == 3 else '5-clases')
name = 'machine_learning/tweeter/base_line'<jupyter_output><empty_output><jupyter_text>Get data<jupyter_code>cine = HtmlParser(200, "http://www.muchocine.net/criticas_ultimas.php", 1)
data_corpus = get_corpus('general-corpus', 'general-corpus', 1, None)

if polarity_dim == 2:
    data_corpus = filter_binary_pn(data_corpus)
    cine = filter_binary_pn(cine.get_corpus())
elif polarity_dim == 3:
    data_corpus = filter_corpus_small(data_corpus)
    cine = filter_corpus_small(cine.get_corpus())
elif polarity_dim == 5:
    cine = cine.get_corpus()
cine = cine[:5000]
used_data = pd.DataFrame(data_corpus)
split = used_data.shape[0] * 0.7
data_corpus = None<jupyter_output>#Intentando obtener datos del archivo csv...
./../Corpus/../data/general-corpus.csv
#Datos recuperados!<jupyter_text>Split data<jupyter_code>used_data.reset_index().groupby('polarity').agg({'index': 'count'}).iplot(kind='bar')
train_corpus = used_data.loc[:split - 1 , :]
test_corpus = used_data.loc[split:, :]<jupyter_output><empty_output><jupyter_text>Initialize ML<jupyter_code>vect = vectorizerIdf if idf else vectorizer
ls = CalibratedClassifierCV(LinearSVC()) if polarity_dim == 2 else OneVsRestClassifier(CalibratedClassifierCV(LinearSVC()))
lr = LogisticRegression(solver='lbfgs') if polarity_dim == 2 else OneVsRestClassifier(LogisticRegression())
mb = MultinomialNB() if polarity_dim == 2 else OneVsRestClassifier(MultinomialNB())
rf = RandomForestClassifier() if polarity_dim == 2 else OneVsRestClassifier(RandomForestClassifier())
pipeline_ls = Pipeline([
    ('vect', copy.deepcopy(vect)),
    ('ls', ls)
])
pipeline_lr = Pipeline([
    ('vect', copy.deepcopy(vect)),
    ('lr', lr)
])
pipeline_mb = Pipeline([
    ('vect', copy.deepcopy(vect)),
    ('mb', mb)
])
pipeline_rf = Pipeline([
    ('vect', copy.deepcopy(vect)),
    ('rf', rf)
])
pipelines = {
    'ls': pipeline_ls,
    'lr': pipeline_lr,
    'mb': pipeline_mb,
    'rf': pipeline_rf
}
pipelines_train = {
    'ls': ls,
    'lr': lr,
    'mb': mb,
    'rf': rf
}<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>folds = pd.read_pickle('../data/pkls/folds_3_clases.pkl') # k-folds precargados
folds = folds.values
x_vect = vect.fit_transform(train_corpus.content, train_corpus.polarity).toarray()
cine_vect = vect.transform(cine.content).toarray()
vect.vocabulary_
results = {}
with tqdm(total=len(clasificadores) * 10) as pbar:
    for c in clasificadores:
        results[c] = { 'real': {}, 'cine_real': {}, 'predicted': {}, 'cine_predicted': {} }
        i = 0
        for train_index, test_index in folds:
            train_x = x_vect[train_index]
            train_y = train_corpus.polarity[train_index]
            test_x = x_vect[test_index]
            test_y = train_corpus.polarity[test_index]

            pipelines_train[c].fit(train_x, train_y)

            predicted = pipelines_train[c].predict(test_x)
            cine_pred = pipelines_train[c].predict(cine_vect)
            
            results[c]['real'][i] = test_y.values.tolist()
            results[c]['cine_real'][i] = cine.polarity.values.tolist()
            
            results[c]['predicted'][i] = predicted.tolist()
            results[c]['cine_predicted'][i] = cine_pred.tolist()
            i = i + 1

            pbar.update(1)

    
pd.DataFrame(results).to_pickle('../results/'+name+'/'+base_dir+'/results.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12375533428165007, 'fraction_numerical': 0.007112375533428165, 'mean_word_length': 4.966060606060606, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '272064', 'license': 'MIT', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1704, 'n_words': 328, 'path': 'Notebooks/Machine Learning - base line-3-clases.ipynb', 'repo_name': 'AdrianInsua/SentimentAnalysis', 'size': '4921'}