<jupyter_start><jupyter_text>Libraries(all libraries you need you can find in requirements.txt)<jupyter_code># MAIN
import pandas as pd
import numpy as np
from tqdm import tqdm_notebook
from collections import Counter
import gc
import pickle
from scripts.storing import *

# NLP
import spacy
import string
import nltk
from nltk.corpus import stopwords
from scripts.cleanup import CleanUpText

# SKLEARN
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, make_scorer
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.manifold import TSNE
import joblib<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\admin\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\admin\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Data loading and Labels to Numbers<jupyter_code>reviews = pd.read_csv('deceptive-opinion.csv')
reviews['deceptive'] = reviews['deceptive'].apply(lambda x: 0 if x == 'truthful' else 1)
reviews['polarity'] = reviews['polarity'].apply(lambda x: 1 if x == 'positive' else 0)
reviews.head(3)<jupyter_output><empty_output><jupyter_text>IT'S A FRANKENSTEIN! (aka Pipeline)P.S. And cross validation F1 score<jupyter_code>sfilter = Pipeline([
    ('cleanup_text', CleanUpText()),
    ('bow', CountVectorizer()),
    ('tfidf', TfidfTransformer(use_idf=True)),
    ('clf', LogisticRegression(solver='lbfgs'))
])
%%time
scores = cross_val_score(sfilter, reviews['text'], reviews['deceptive'], scoring=make_scorer(f1_score), cv=10)
print("Average F1 score (by 10 folds): %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
_ = sfilter.fit(reviews['text'], reviews['deceptive'])<jupyter_output><empty_output><jupyter_text>Model storing<jupyter_code>save_model(sfilter, 'models/sfilter.pkl')<jupyter_output>Model was saved in models/sfilter.pkl
Use load_model to load model from file.
{'fraction_non_alphanumeric': 0.11394712853236098, 'fraction_numerical': 0.008204193254329991, 'mean_word_length': 6.89568345323741, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '638414', 'license': 'MIT', 'n_tokens_mistral': 783, 'n_tokens_neox': 749, 'n_words': 181, 'path': 'spam-filter-fitting.ipynb', 'repo_name': 'Shemka/ReviewsScoring', 'size': '2194'}