<jupyter_start><jupyter_text>Prototype stopped: Performance too bad. Try other preprocessing approach in next iteration. Sentiment Analysis PrototypingPrototype code for ML-based sentiment analysis. Classification case neg/non-negative only.<jupyter_code>import os, sys, re, string, pickle, random
sys.path.append("..")
from config import credentials
import dropbox

import numpy as np
import pandas as pd

import nltk
nltk.data.path.append("../data/external/nltk_data")
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.cistem import Cistem

import emoji

from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import FunctionTransformer
from sklearn.model_selection import train_test_split, GridSearchCV, cross_validate, LeaveOneOut
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import MultinomialNB, GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, balanced_accuracy_score
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

from yellowbrick.classifier import ConfusionMatrix, ClassificationReport, PrecisionRecallCurve, ROCAUC

import matplotlib.pyplot as plt
%matplotlib inline
random_state = 23
random.seed(random_state)<jupyter_output><empty_output><jupyter_text>Loading<jupyter_code>team_dbx = dropbox.DropboxTeam(credentials.dropbox_team_access_token)
team_root = team_dbx.with_path_root(dropbox.common.PathRoot.namespace_id(
    credentials.dropbox_team_namespace_id))
user_dbx = team_root.as_user(credentials.dropbox_team_member_id)

data_path = "/Data/CSVData"
fpath = os.path.join(data_path, "TestData", "forSentAnalysis.csv")

_, res = user_dbx.files_download(fpath)
labeled_data = pd.read_csv(res.raw)
labeled_data.shape<jupyter_output><empty_output><jupyter_text>Rating schema: {0: "positive", 10: "neutral", 20: "negative", 30: "offensive", -2: "notAssessable"}  Preprocessing Dataset cleansing<jupyter_code># Remove unnecessary cols and rename them uniformly
labeled_data.drop(["id", "Level", "Topic", "sentiment"], axis=1, inplace=True)
labeled_data.columns = ["source", "text", "rating"]

# Remove not assessable
labeled_data = labeled_data[labeled_data.rating != -2]

# Remove empty texts
labeled_data.text.replace("", np.nan, inplace=True)
labeled_data.dropna(subset=["text"], inplace=True)
labeled_data = labeled_data[~labeled_data.text.str.isspace()]

# Remove duplicated texts
labeled_data.drop_duplicates(subset=["text"], keep="first", inplace=True)

# Remap rating labels
new_rating_schema = {0: 1, 10: 1, 20:-1, 30:-1}
labeled_data.rating = labeled_data.rating.map(new_rating_schema)

labeled_data.shape
# from IPython.core.display import HTML
# HTML(labeled_data[["text", "rating"]].tail(100).to_html())<jupyter_output><empty_output><jupyter_text>New rating schema: {1: "non-negative", -1: "negative"}<jupyter_code>labeled_data.rating.value_counts()<jupyter_output><empty_output><jupyter_text>Text cleansing<jupyter_code>stopwords_list = stopwords.words("german")
stemmer = Cistem()

def clean_text(text):
    """ Util: Cleans text string.
    > Lowercase string
    > Punctuation removal
    > Replace numbers with "NUM" placeholder
    > Removes urls
    > Split emojis away from words and each other
    > Stopwords removal
    > Stemming
    """
    lowercased = text.lower()
    split_emoji = emoji.get_emoji_regexp().split(lowercased) # splits emoji away
    punctuations = string.punctuation + "„" + "”" + "–" + "‘" + "’" + "“" + "…" + "‼"
    punct_removed = " ".join(split_emoji).translate(str.maketrans("", "", punctuations))
    num_replaced = re.sub(r"\b\d+\b", "NUM", punct_removed)
    urls_removed = re.sub(r"http\S+", "", num_replaced)
    tokenized = word_tokenize(urls_removed) 
    cleaned_tokens = [word for word in tokenized if word not in stopwords_list]
    stemmed = [stemmer.stem(word) for word in cleaned_tokens]
    cleaned_txt = " ".join(stemmed)
    
    return cleaned_txt


# def get_emoji_free(text):
#     return emoji.get_emoji_regexp().sub(r"", text)

# def clean_text(text):
#     """ Util: Cleans text string.
#     > Lowercase string
#     > Replace game scores with "GAME_SCORE" placeholder
#     > Punctuation removal
#     > Replace numbers with "NUM" placeholder
#     > Split emojis away from words and each other
#     > Stopwords removal
#     > Stemming
#     > Removes single characters
#     """
#     lowercased = text.lower()
#     emoji_free = get_emoji_free(lowercased)
#     punctuations = string.punctuation + "„" + "”" + "–" + "‘" + "’" + "“" + "…" + "‼"
#     punct_removed = emoji_free.translate(str.maketrans("", "",
#                                                            punctuations))
#     num_removed = re.sub(r"\b\d+\b", "", punct_removed)
#     urls_removed = re.sub(r"http\S+", "", num_removed)
#     tokenized = word_tokenize(urls_removed) 
#     cleaned_tokens = [word for word in tokenized if word not in stopwords_list]
#     stemmed = [stemmer.stem(word) for word in cleaned_tokens]
#     cleaned_txt = " ".join([word for word in stemmed if len(word) > 1])
    
#     return cleaned_txt


# Apply text cleansing and drop rows with empty cleaned_txt vals
labeled_data["cleaned_txt"] = labeled_data.text.apply(clean_text)
labeled_data.drop(labeled_data[labeled_data.cleaned_txt == ""].index, inplace=True)
#labeled_data[["text", "cleaned_txt", "rating"]].to_csv(os.path.join(data_path, "TestData", "labeled_sentiment.csv"), index=False)<jupyter_output><empty_output><jupyter_text>Exploration NaN Stats<jupyter_code>labeled_data.isna().sum()<jupyter_output><empty_output><jupyter_text>Duplicated texts (after cleaning)<jupyter_code>labeled_data.text.duplicated().value_counts()<jupyter_output><empty_output><jupyter_text>Distribution of sources<jupyter_code>print("Sources count:", len(labeled_data.source.value_counts()))
print("Avg count of doc per source:", np.mean(labeled_data.source.value_counts().tolist()))
print("Top 5 sources:")
labeled_data.source.value_counts()[:5]<jupyter_output>Sources count: 66
Avg count of doc per source: 15.909090909090908
Top 5 sources:<jupyter_text>Distribution of ratings<jupyter_code>labeled_data.rating.value_counts(sort=False).plot(kind="bar")<jupyter_output><empty_output><jupyter_text>-> we have more or less balanced (but generally too less) data.  Sentiment AssignmentML-based approach.  Train/Test-Split No need for Dev-Set, as we use cross-validation for model tuning.<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(labeled_data.cleaned_txt, labeled_data.rating,
                                                    test_size=0.2, stratify=labeled_data.rating, 
                                                    random_state=random_state)

print("X shapes (train/test):", X_train.shape, "/", X_test.shape)
print("y shapes (train/test):", y_train.shape, "/", y_test.shape)<jupyter_output>X shapes (train/test): (840,) / (210,)
y shapes (train/test): (840,) / (210,)<jupyter_text>Vectorizer<jupyter_code>count_vect = CountVectorizer(lowercase=False, # already lowercased
                             tokenizer=word_tokenize,
                             #ngram_range=(1,2),
                             max_features=4000) # tune max_features for generalization eventually, 3gram 25k, 2gram 10k

tfidf = TfidfVectorizer(lowercase=False, # already lowercased
                        tokenizer=word_tokenize,
                        #ngram_range=(1,2),
                        max_features=4000) # tune max_features for generalization eventually, 3gram 25k, 2gram 10k<jupyter_output><empty_output><jupyter_text>Models<jupyter_code>logreg = LogisticRegression(solver="liblinear", max_iter=4000, random_state=random_state, n_jobs=1)
nb = MultinomialNB()
rf = RandomForestClassifier(random_state=random_state, n_jobs=-1)

models = [logreg, nb]<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>params = {}
params["LogisticRegression"] = {"penalty": ["l2"],
                                "dual": [True],
                                "C": [0.01, 0.1, 1., 10., 50., 100., 200.],
                                "class_weight": [None, "balanced"]}
params["MultinomialNB"] = {"alpha": [1.0e-10, 1.],
                           "fit_prior": [True, False]}
# params["RandomForestClassifier"] = {"n_estimators": [2, 32, 64, 128, 256],
#                                     "criterion": ["gini", "entropy"],
#                                     "min_samples_leaf": [1, 2],
#                                     "max_features": ["auto"],
#                                     "class_weight": [None, "balanced"]}<jupyter_output><empty_output><jupyter_text>Pipeline<jupyter_code>classifiers = {}
scoring = "f1"

# def get_post_length(txt):
#     """ Util: Length of text """
#     return np.array([len(t) for t in txt]).reshape(-1, 1)

for model in models:
    classifiers[type(model).__name__ + "_pipe"] = Pipeline([
        ("count_vect", count_vect),
        #("tfidf", tfidf),
        ("clf", GridSearchCV(model, param_grid=params[type(model).__name__],
                             scoring=scoring, cv=8, refit=True, n_jobs=-1))
    ])
#     classifiers[type(model).__name__ + "_pipe"] = Pipeline([
#         ("features", FeatureUnion([
#             ("text", Pipeline([
#                 ("count_vect", count_vect),
#                 #("tfidf", tfidf)
#             ])),
#             ("text_length", Pipeline([
#                 ("txt_len", FunctionTransformer(get_post_length, validate=False)),
#             ]))
#         ])),
#         ("clf", GridSearchCV(model, param_grid=params[type(model).__name__],
#                              scoring=scoring, cv=8, refit=True, n_jobs=-1))
#     ])<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>for _, clf_pipeline in classifiers.items():
    clf_pipeline.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Score<jupyter_code>best_score = 0
best_model_pipe = None # = best generalization

for name, clf_pipeline in classifiers.items():
    y_pred = clf_pipeline.predict(X_test)
    scores = cross_validate(clf_pipeline, X_test, y_test, scoring=scoring, cv=8, n_jobs=1)
    mean_score = scores["test_score"].mean()
    mean_fit_time = scores["fit_time"].mean()
    
    print(name)
    print("Test score (mean):", mean_score)
    print("Test fit time (mean):", mean_fit_time)
    print("Train score:", clf_pipeline.named_steps["clf"].best_score_)
    print("Params:", clf_pipeline.named_steps["clf"].best_params_, "\n")
    
    if mean_score > best_score:
        best_model_pipe = clf_pipeline
        best_score = mean_score
#len(best_model_pipe.named_steps["count_vect"].get_feature_names())
#best_model_pipe.named_steps["count_vect"].vocabulary_<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>print("Best pipeline CLF:", best_model_pipe.named_steps["clf"].best_estimator_)
class_encoding = {1: "non-negative", -1: "negative"}<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>cm = ConfusionMatrix(best_model_pipe, is_fitted=True, encoder=class_encoding)
cm.score(X_test, y_test)
cm.poof()<jupyter_output>/home/datadonk23/anaconda3/envs/WBNRW19/lib/python3.7/site-packages/yellowbrick/classifier/base.py:232: YellowbrickWarning: could not determine class_counts_ from previously fitted classifier
  YellowbrickWarning,<jupyter_text>Classification Report<jupyter_code>print(classification_report(y_test, best_model_pipe.predict(X_test)))

# Visualizer
report = ClassificationReport(best_model_pipe, is_fitted=True, encoder=class_encoding, support=True)
report.score(X_test, y_test)
report.poof()<jupyter_output>precision    recall  f1-score   support

          -1       0.62      0.37      0.46       101
           1       0.57      0.79      0.66       109

    accuracy                           0.59       210
   macro avg       0.59      0.58      0.56       210
weighted avg       0.59      0.59      0.57       210<jupyter_text>Precision-Recall Curve<jupyter_code>pr_curve = PrecisionRecallCurve(best_model_pipe)
pr_curve.fit(X_train, y_train)
pr_curve.score(X_test, y_test)
pr_curve.poof()<jupyter_output><empty_output><jupyter_text>ROCAUC<jupyter_code>rocauc = ROCAUC(best_model_pipe, is_fitted=True, encoder=class_encoding)
rocauc.score(X_test, y_test)
rocauc.poof()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12481119325860561, 'fraction_numerical': 0.016614993242706096, 'mean_word_length': 4.09518023491292, 'pattern_counts': {'":': 11, '<': 81, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '456405', 'license': 'MIT', 'n_tokens_mistral': 4398, 'n_tokens_neox': 4183, 'n_words': 770, 'path': 'notebooks/1.3-tte-sentiment-prototyping.ipynb', 'repo_name': 'viennadatasciencegroup/data4good-wahlbeobachtung19', 'size': '12579'}