<jupyter_start><jupyter_text>Se quitan instancias con muy poco texto, se decide en base a la longitud del texto respecto a la longitud del título promedio<jupyter_code>df.head()
df.title.apply(len).mean()
# Identificar indices donde se da una longitud de texto menor a la longitud del título promedio
df.drop(df.loc[df.text.apply(len) <= 78].index, inplace = True)
df.info()
conteo_categorias=df['label'].value_counts()
tabla_categorias=pd.DataFrame()
tabla_categorias['conteo']=conteo_categorias
tabla_categorias['porcentaje']=(conteo_categorias/df.shape[0]*100)
tabla_categorias['acumulado']=tabla_categorias['porcentaje'].cumsum()
# (0 = fake and 1 = real).
tabla_categorias<jupyter_output><empty_output><jupyter_text>Preparación<jupyter_code>import re
import nltk
import numpy as np
import matplotlib.pyplot as plt
#nltk.download('stopwords')
#nltk.download('words')
stopwords_nltk = set(nltk.corpus.stopwords.words('english'))
df['text'][30]
%time datos = df.rename(columns={'text':'tokens'})
datos = datos.drop('Unnamed: 0',axis=1)<jupyter_output><empty_output><jupyter_text>Tokenización<jupyter_code># %time datos['tokens'].sample(100)=datos['tokens'].apply(nltk.word_tokenize)
%time datos['tokens']=datos['tokens'].apply(nltk.word_tokenize)
datos.head()
def mostrar_frecuencias(datos):
    tokens_concatenate=np.concatenate(datos['tokens'])
    fdist = nltk.FreqDist(tokens_concatenate)
    topwords = fdist.most_common(20)
    x,y = zip(*topwords)
    print('Numero de tokens:',len(fdist))
    for i,token in enumerate(topwords[0:20]):
        print(i+1,token)
    plt.figure(figsize=(8,6))
    plt.title("Frecuencias de Palabras para Noticias " + str(var))
    plt.bar(x,y)
    plt.xticks(rotation=90)
    plt.show()<jupyter_output><empty_output><jupyter_text>Se muestran los tokens sin limpiar para resaltar la importancia de este proceso, más adelante se mostrarán los tokens para noticias falsas y reales<jupyter_code># Se muestra frecuencia de noticias reales
var = "Reales"
%time mostrar_frecuencias(datos.loc[datos['label'] == 0].reset_index(drop=True))
# Se muestra frecuencia de noticias falsas
var = "Falsas"
%time mostrar_frecuencias(datos.loc[datos['label'] == 0].reset_index(drop=True))<jupyter_output>Numero de tokens: 22726
1 (',', 15519)
2 ('the', 14486)
3 ('.', 11393)
4 ('to', 7587)
5 ('of', 7193)
6 ('a', 6505)
7 ('and', 6205)
8 ('in', 5623)
9 ('’', 3453)
10 ('that', 3370)
11 ('s', 3193)
12 ('”', 2795)
13 ('“', 2793)
14 ('on', 2763)
15 ('said', 2680)
16 ('for', 2514)
17 ('is', 2081)
18 ('The', 1810)
19 ('was', 1765)
20 ('with', 1743)<jupyter_text>Limpieza de tokens<jupyter_code>import gensim
from gensim.parsing.preprocessing import remove_stopwords, STOPWORDS
stop_words = nltk.corpus.stopwords.words('english')
def limpiar_tokens(tokens):
    # Esta función quita todos los caracteres que no sean alfabeticos
    tokens=[re.sub(r'[0-9\. ]+','',token) for token in tokens] # quitar números
    tokens=[re.sub(r'[^A-Za-z]+','',token) for token in tokens] # quitar otros caracteres
    #tokens=[token for token in tokens if token not in stopwords_nltk]
    tokens=[token for token in tokens if token not in stop_words]
    tokens=[token.lower() for token in tokens if len(token)>2]
    return tokens
%time datos['tokens']=datos['tokens'].apply(limpiar_tokens)<jupyter_output>CPU times: total: 2.11 s
Wall time: 2.14 s<jupyter_text>Stemming<jupyter_code>stemmer=nltk.stem.SnowballStemmer("english")
%time datos['tokens']=datos['tokens'].apply(lambda tokens: [stemmer.stem(w) for w in tokens])<jupyter_output>CPU times: total: 3.12 s
Wall time: 3.14 s<jupyter_text>Limpieza adicional<jupyter_code># Extensión de stopwords
stop_words.extend(['the','said','trump'])
%time datos['tokens']=datos['tokens'].apply(limpiar_tokens)<jupyter_output>CPU times: total: 1.25 s
Wall time: 1.25 s<jupyter_text>Se muestra como cambia las frecuencias de datos el proceso de stemming y remoción de caracteres no alfabeticos<jupyter_code>var = "Reales"
%time mostrar_frecuencias(datos.loc[datos['label'] == 1].reset_index(drop=True))
var = "Falsas"
%time mostrar_frecuencias(datos.loc[datos['label'] == 0].reset_index(drop=True))<jupyter_output>Numero de tokens: 12824
1 ('state', 875)
2 ('would', 824)
3 ('presid', 747)
4 ('year', 673)
5 ('new', 643)
6 ('one', 622)
7 ('peopl', 566)
8 ('republican', 520)
9 ('say', 510)
10 ('also', 489)
11 ('elect', 468)
12 ('govern', 465)
13 ('nation', 451)
14 ('clinton', 451)
15 ('time', 449)
16 ('democrat', 428)
17 ('like', 424)
18 ('could', 416)
19 ('hous', 411)
20 ('countri', 407)<jupyter_text>Se aprecia la mejora en la calidad de las palabaras para el BoW  Implementación de Algoritmos<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from ast import literal_eval
import ast
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.metrics import confusion_matrix,accuracy_score,classification_report, recall_score, f1_score, precision_recall_fscore_support
datos = datos.reset_index(drop=True)
datos.head()
datos.info()
def literal_return(val):
    try:
        return ast.literal_eval(val)
    except ValueError:
        return (val)

datos.tokens=datos['tokens'].apply(literal_return)<jupyter_output><empty_output><jupyter_text>Matriz TF-IDF<jupyter_code>%time fdist = nltk.FreqDist(np.concatenate(datos['tokens'].reset_index(drop=True))) #].sampe(10000)
%time tokens=fdist.most_common(len(fdist))
%time tokens_tf=pd.DataFrame(tokens,columns=['token','TF'])
fdist
tokens_tf.shape
tokens_tf.query("TF>1",inplace=True)
tokens_tf.shape
tokens_tf.head()
query="TF<=100000 and TF>=1"
tokens_tf.query(query)
bow = tokens_tf.query(query).token.values
bow.shape
bow
%time tfidf=pd.DataFrame(TfidfVectorizer(vocabulary=bow).fit_transform(datos['tokens'].str.join(" ")).toarray(), columns=bow)
tfidf
X_train, X_test, y_train, y_test = train_test_split(
    tfidf.values, datos.label.values, test_size=0.2, random_state=42)
cms_train=[]
cms_test=[]
accuracy_train = []
accuracy_test = []
f1_train = []
f1_test = []
recall_train = []
recall_test = []
reporte_train = []
reporte_test = []
max_depths = np.arange(2,22,2) #(10,110, 10)
for max_depth in max_depths:
    print("max_depth:",max_depth)
    tree = DecisionTreeClassifier(max_depth=max_depth,class_weight='balanced')
    tree.fit(X_train, y_train)

    predicciones_train=tree.predict(X_train)
    predicciones_test=tree.predict(X_test)

    cms_train.append(confusion_matrix(y_train,predicciones_train))
    cms_test.append(confusion_matrix(y_test,predicciones_test))
    accuracy_train.append(accuracy_score(y_train,predicciones_train))
    accuracy_test.append(accuracy_score(y_test,predicciones_test))
    f1_train.append(f1_score(y_train,predicciones_train,average='weighted'))
    f1_test.append(f1_score(y_test,predicciones_test,average='weighted'))
    recall_train.append(recall_score(y_train,predicciones_train,average='weighted'))
    recall_test.append(recall_score(y_test,predicciones_test,average='weighted'))
    reporte_train.append(precision_recall_fscore_support(y_train,predicciones_train))
    reporte_test.append(precision_recall_fscore_support(y_test,predicciones_test))
    
    print("Train:")
    print(cms_train[-1])
    print(classification_report(y_train,predicciones_train))

    print("Test:")
    print(cms_test[-1])
    print(classification_report(y_test,predicciones_test))
    print("-----------")
fig,ax=plt.subplots(2,1,figsize=(10,6),tight_layout=True)
fig.suptitle("     DecisionTree")
ax[0].scatter(max_depths,accuracy_train,s=50,alpha=0.8, label = 'Accuracy')
ax[0].scatter(max_depths,f1_train,s=50,alpha=0.8, label = 'F1-score')
ax[0].scatter(max_depths,recall_train,s=50,alpha=0.8, label = 'Recall')
ax[0].legend()
ax[0].grid()
ax[0].set_title('\n\nTraining')
# ax[0].set_xlabel('max_depth')

ax[1].scatter(max_depths,accuracy_test,s=50,alpha=0.8, label = 'Accuracy')
ax[1].scatter(max_depths,f1_test,s=50,alpha=0.8, label = 'F1-score')
ax[1].scatter(max_depths,recall_test,s=50,alpha=0.8, label = 'Recall')
ax[1].legend()
ax[1].grid()
ax[1].set_title('Test')
ax[1].set_xlabel('max_depth')

plt.show()<jupyter_output><empty_output><jupyter_text>Validación Cruzada para el modelo out of sample<jupyter_code>from sklearn.model_selection import cross_validate
#datos = pd.read_csv('data_equilibrada.csv')
#datos.tokens=datos.tokens.apply(literal_eval)
#bow = pd.read_csv('bow.csv')
#tfidf=pd.DataFrame(TfidfVectorizer(vocabulary=bow.token.values).fit_transform(datos['tokens'].str.join(" ")).toarray(), columns=bow.token.values)
X=tfidf.values
y=datos.label.values
dt = DecisionTreeClassifier(max_depth=8,class_weight='balanced')
cv_dt=cross_validate(dt, X, y, cv=20,scoring=('accuracy','f1_weighted','recall_weighted'),n_jobs=-1)
pd.DataFrame(cv_dt).describe()
plt.hist(cv_dt['test_accuracy'],bins=6)
plt.title("DecissionTree")
plt.xlabel("test_accuracy ")
plt.show()<jupyter_output><empty_output><jupyter_text>Calibración de Hiperparámetros<jupyter_code># Import moduels for Hyperparameter Tuning
from scipy.stats import randint
from sklearn.model_selection import RandomizedSearchCV
# Setup the parameters and distributions to sample from: param_dist
param_dist = {"max_depth": [3, 30], "min_samples_leaf": randint(1,20), "criterion": ["gini", "entropy"], "splitter": ["best", "random"]}
# Instantiate a Decision Tree classifier: tree
tree = DecisionTreeClassifier()
# Instantiate the RandomizedSearchCV objetc: tree_cv, cv is k-folds.
tree_cv = RandomizedSearchCV(tree, param_dist, cv=5)
# Fit it to the data
tree_cv.fit(X_train, y_train)
# Print the tunred parameters and score
print("Tuned Decision Tree Parameters: {}".format(tree_cv.best_params_))
print("Best score is {}".format(tree_cv.best_score_))
predicciones_test2=tree_cv.predict(X_test)
accuracy_score(y_test,predicciones_test2)
confusion_matrix(y_test,predicciones_test2)
accuracy_score(y_test,predicciones_test2)
f1_score(y_test,predicciones_test2,average='weighted')
recall_score(y_test,predicciones_test2,average='weighted')
precision_recall_fscore_support(y_test,predicciones_test2)
print("Test:")
print(cms_test[-1])
print(classification_report(y_test,predicciones_test))
print("-----------")<jupyter_output>Test:
[[78 17]
 [14 85]]
              precision    recall  f1-score   support

           0       0.85      0.82      0.83        95
           1       0.83      0.86      0.85        99

    accuracy                           0.84       194
   macro avg       0.84      0.84      0.84       194
weighted avg       0.84      0.84      0.84       194

-----------
{'fraction_non_alphanumeric': 0.14622243078366964, 'fraction_numerical': 0.040732050680431724, 'mean_word_length': 6.430962343096234, 'pattern_counts': {'":': 4, '<': 45, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '146263', 'license': 'Apache-2.0', 'n_tokens_mistral': 4299, 'n_tokens_neox': 4050, 'n_words': 735, 'path': '1_supervised/Supervisado.ipynb', 'repo_name': 'jonyzp/master-datos-ml-kjjj', 'size': '10655'}