<jupyter_start><jupyter_text>UserBasicClassifierUn classificador básico con la matadata del usuario<jupyter_code>import pandas as pd
import numpy as np

import os
import pickle

from sklearn.ensemble import BaggingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model  import LogisticRegression

from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.model_selection import train_test_split

from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from catboost import CatBoostClassifier

from sklearn.pipeline import Pipeline
from sklearn.base import TransformerMixin
users = pd.read_pickle('trainDatasets/users_dataset.pkl')
users = users[(users.user_type=='Human') | (users.user_type=='Spambot_2')]
users.head()
def transform_x(df):
    df.loc[:, 'crawled_at'] = pd.DatetimeIndex(pd.to_datetime(df.crawled_at, utc=None)).tz_convert(tz='UTC')
    df.loc[:, 'created_at'] = pd.DatetimeIndex(pd.to_datetime(df.created_at, utc=None)).tz_convert(tz='UTC')
    account_life = df.crawled_at - df.created_at
    account_life_meses = np.round(account_life.dt.days / 30)
    account_life_meses = account_life_meses.replace(0, 1)
    account_life_semanas = np.round(account_life.dt.days / 7)
    account_life_semanas = account_life_semanas.replace(0, 1)
    numValues = ['tweet_count', 'listed_count', 'followers_count', 'following_count']
    for i in numValues:
        colName_mes = i+'_mes'
        df.loc[:,colName_mes] = df[i] / account_life_meses

        colName_semana = i+'_semana'
        df.loc[:,colName_semana] = df[i] / account_life_semanas

    df.loc[:,'verified'] = ~df.verified.isna()
    df.loc[:,'description'] = ~df.description.isna()
    df.loc[:,'location'] = ~df.location.isna()
    df.loc[:,'url'] = ~df.url.isna()

    columnas = np.concatenate(
        [[x+'_mes', x+'_semana'] for x in numValues]
    )
    columnas = np.concatenate([
        [
            'verified',
            'description',
            'location',
            'url',
        ],
        columnas
    ])
    df = df[columnas]
    df = df.fillna(0)
    return df
    
#     def fit(self, x, y=0):
#         return self
#     def transform(self, x, y=0):
#         x = self.getVars(x)
#         return x
#     def fit_transform(self, x, y=0):
#         x = self.getVars(x)
#         return x
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('cat', CatBoostClassifier())
])

le = preprocessing.LabelEncoder()
x = transform_x(users)
y =  le.fit_transform(users['bot'])

X_train, X_test, y_train, y_test = train_test_split(x, y)

# pipeline.fit(X_train, y_train)
y_predicted_cat = pipeline.predict_proba(X_test)
y_pred = pipeline.predict(X_test)

from sklearn.metrics import roc_auc_score
cat_auc = roc_auc_score(y_test,y_predicted_cat[:,1])
print("El valor del AUC es: ", cat_auc)
from scikitplot.metrics import plot_roc
%matplotlib inline
plot_roc(y_test,y_predicted_cat, plot_micro = False, plot_macro= False)
# Matriz de Confusión
conf = confusion_matrix(y_test, y_pred)
predicted_cols = ['pred_'+str(c) for c in le.classes_]
display(pd.DataFrame(conf, index= ['real_Human','real_Bot'],columns = ['pred_Human','pred_Bot']))
print(classification_report(y_test, y_pred))
# Re-Entrenar con el dataset completo
modelo = pipeline.fit(transform_x(users), y)
# # Exportar Modelo a archivo binario
filename = 'modelPipeline.pkl'
with open(filename, 'wb') as twModel:
    pickle.dump(modelo, twModel)
import lime
import lime.lime_tabular
explainer = lime.lime_tabular.LimeTabularExplainer(
    X_train, 
    feature_names=X_train.columns,
    class_names=le.classes_, 
    discretize_continuous=False
)

# X_train, X_test, y_train, y_test
i = np.random.randint(0, X_test.shape[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11796913887506222, 'fraction_numerical': 0.0037332005973120955, 'mean_word_length': 4.528198074277854, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '137724', 'license': 'MIT', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1412, 'n_words': 283, 'path': 'modelo/generacionModelo.ipynb', 'repo_name': 'fermasia/argBot', 'size': '4018'}