<jupyter_start><jupyter_text>---  Load data<jupyter_code>pwd
CORPUS = []
LABEL = []
# Step 1: get all the fake news text files into a list named corpus, give them label 1 -- fake
dir1 = 'Data/size_normalized_texts/fake/'
list = os.listdir(dir1)
number_files = len(list)
for i in range(number_files):
        title = list[i]
        LABEL.append(1)
        with open(dir1 + title,'r') as reader:

            doc = reader.read()
            doc.lower()
            doc.split()
            reader.close
            CORPUS.append(doc)
assert len(CORPUS) == len(LABEL)
len(CORPUS)
# Step 2: repeat the procedure with true news, give them label 0 -- true
dir2 = 'Data/size_normalized_texts/true/'
list = os.listdir(dir2)
number_files = len(list)
for i in range(number_files):
        title = list[i]
        LABEL.append(0)
        with open(dir2 + title,'r') as reader:

            doc = reader.read()
            doc.lower()
            doc.split()
            reader.close
            CORPUS.append(doc)
assert len(CORPUS) == len(LABEL)<jupyter_output><empty_output><jupyter_text>Vectorize words<jupyter_code>#stop words in portuguese br:
custom = ['como', 'um', 'que', 'para', 'como', 'uma', 'de', 'ou', 'os', 'em', 'por', 'ma',
                 'se', 'era', 'ela', 'ele', 'era', 'seu', 'sua', 'mai', 'mas', 'nos', 'isso', 'mas', 'entre', 
                 'entre', 'outro', 'na', 'até', 'sobre', 'essa', 'esse', 'quando', 'mais', 'ao', 'dos', 'tem', 
                  'dos', 'está', 'não', 'da', 'disse', 'diz', 'foi', 'caso', 'outra', 'também', 'já', 'ano', 
                'ainda', 'são', 'anos', 'sem', 'porque', 'há', 'eles', 'segundo', 'afirmou', 'acordo', 'muito', 
               'todo', 'muito', 'conta', 'mesmo', 'dele', 'todos', 'dia', 'depois', 'então', 'tudo', 'seja', 'quem'
               'toda', 'após', 'só', 'agora', 'hoje', 'foram', 'grupo', 'assim', 'fazer', 'desde', 'eu', 'estava',
               'estão', 'durante', 'onde', 'das', 'vai', 'onde', 'deu', 'quem', 'qual', 'toda', 'ante', 'outros', 
               'pela', 'fez', 'pelo', 'tinha']
def vectors_and_df(corpus, label):
    """
    creates vectors for the text files and returns dataframe with articles as word vectors 
    """
    
    cv = TfidfVectorizer()
    cv.fit(corpus)
    corpus_vecs = cv.transform(corpus)
    
    return pd.DataFrame(corpus_vecs.todense(), index=label, 
                        columns=cv.get_feature_names()), cv
# Store results into dataframe, keep cv for later prediction
df, cv = vectors_and_df(CORPUS, LABEL)<jupyter_output><empty_output><jupyter_text>Split and train data<jupyter_code># Define features and target column
X = df
y = df.index
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.2)<jupyter_output><empty_output><jupyter_text>Try different models Improve logistic regression only by increasing C? Any better way to get better parameters?<jupyter_code>MODELS = {
    "MultinomialNB": {"alpha": 0.005},
    "CategoricalNB": {"alpha": 0.01},
    "RandomForestClassifier": {
        "n_estimators": 500,
        "max_depth": 200,
        "max_features": "auto",
        "n_jobs": -1,
        "random_state": 1,
    },
    "LogisticRegression": {"C": 2e7},
}

def train_models(models_params):
    """trains models on corpus and returns dataframe with scores"""
    
    scores = {}
    for model in models_params:
        if model == "LogisticRegression":
            m = LogisticRegression(**models_params[model])
        elif model == "RandomForestClassifier":
            m = RandomForestClassifier(**models_params[model])
        elif model == "MultinomialNB":
            m = MultinomialNB(**models_params[model])
        elif model == "CategoricalNB":
            m = MultinomialNB(**models_params[model])

        m.fit(Xtrain, ytrain)
        score_train = m.score(Xtrain, ytrain)
        score_test = m.score(Xtest, ytest)
        scores[f"{model}"] = {
            "params": models_params[model],
            "train score": score_train,
            "test score": score_test,
            }
    return pd.DataFrame(scores).T
train_models(MODELS)<jupyter_output><empty_output><jupyter_text>Evaluate best performing model<jupyter_code># Train on most promising model
model = "LogisticRegression"
m = LogisticRegression(**MODELS[model], max_iter=300)
m.fit(Xtrain, ytrain)
# Create the predictions for Y training data
preds = m.predict(Xtest)

print(m.score(Xtrain, ytrain))
print(m.score(Xtest, ytest))<jupyter_output>1.0
0.9284722222222223<jupyter_text>Confusion Matrix<jupyter_code># Create a confusion matrix
cnf_matrix = metrics.confusion_matrix(ytest, preds)
# name  of classes
class_names=[0,1] 

# Set fig and axes 
fig, ax = plt.subplots()
tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks, class_names)
plt.yticks(tick_marks, class_names)

# Create heatmap
sns.heatmap(pd.DataFrame(cnf_matrix), annot=True, cmap="YlGnBu" ,fmt='g')
ax.xaxis.set_label_position("top")
plt.tight_layout()
plt.title('Confusion matrix', y=1.1)
plt.ylabel('Actual label')
plt.xlabel('Predicted label')
plt.savefig('confusion_matrix.png', dpi=200, bbox_inches='tight', pad_inches=0.5)
# Assign True Neg, False Pos, False Neg, True Pos variables
cnf_matrix = np.array(cnf_matrix).tolist()

tn_fp, fn_tp = cnf_matrix

tn, fp = tn_fp
fn, tp = fn_tp
# Print Scores 

print("Accuracy:",round(metrics.accuracy_score(ytest, preds)*100, 2),'%')
print("Precision:",round(metrics.precision_score(ytest, preds)*100, 2), '%')
print("Recall:",round(metrics.recall_score(ytest, preds)*100, 2), '%')
print("Specificity:", round((tn/(tn+fp))*100, 2), '%')
print("Misclassification Rate:", round((fp+fn)/(tn+fp+fn+tn)*100, 2), '%')<jupyter_output>Accuracy: 92.85 %
Precision: 92.55 %
Recall: 93.19 %
Specificity: 92.5 %
Misclassification Rate: 7.18 %<jupyter_text>Cross-Validation<jupyter_code>accuracy = cross_val_score(m, Xtrain, ytrain, cv=5, scoring='accuracy')
print("cross-validation scores", accuracy)<jupyter_output>cross-validation scores [0.92881944 0.90538194 0.91493056 0.90017361 0.9140625 ]<jupyter_text>Train the entire dataset on the chosen model<jupyter_code>model = "LogisticRegression"
m = LogisticRegression(**MODELS[model], max_iter=200)
m.fit(X, y)<jupyter_output><empty_output><jupyter_text>Create prediction function<jupyter_code>def predict(new_text):
    
    """
    Takes the pre-trained model pipeline and predicts new artist based on unseen text.
    
    Parameters
    ----------
    model : Trained scikit-learn model pipeline.
    new_text : str
    
    Returns
    ---------
    prediction : str
    
    """
    article = [new_text]
    # transform song into vector matrix
    new_article_vecs = cv.transform(article)
    ynew = new_article_vecs.todense()
    
    prediction = m.predict(ynew)
    
    return prediction[0]
if __name__ == '__main__':
    # Whatever happens after this line, execute it when running "python lyrics_classifier.py"
    # and DO NOT execute these lines of code if things from this script are imported from other scripts.
    
    user_input = input('Please Enter Some Text: ')

    prediction = predict(user_input)
    print('Here is your prediction!')
    print(prediction)<jupyter_output>Please Enter Some Text: test
Here is your prediction!
1
{'fraction_non_alphanumeric': 0.150711424229866, 'fraction_numerical': 0.020721094073767096, 'mean_word_length': 3.6679561573178594, 'pattern_counts': {'":': 19, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '554523', 'license': 'MIT', 'n_tokens_mistral': 2672, 'n_tokens_neox': 2498, 'n_words': 596, 'path': 'NLP_model.ipynb', 'repo_name': 'brauliotegui/FAKE', 'size': '7239'}