<jupyter_start><jupyter_text>Pasos del TP*   Explorar los datos*   Plantear el problema a resolver*   Preprocesar los datos a un formato adecuado*   Elegir algoritmos*   Fittear y validar*   Decidir el algoritmo final, y testear Problemas que se buscan resolver:Se ha demostrado que cada vez más empresas (especialmente empresas de comercio electrónico) tienen grandes dificultades en la conversión de propects (clientes potencialess) a clientes activos con la primer compra, además sostener a los clientes actuales en la dinámica e interacción en el tiempo con los productos y la empresa suele ser de difícil comprensión, generando pérdidas de facturación por cancelaciones hasta la pérdida del clienteLas investigaciones se han centrado en el análisis del producto y el ciclo de valor del cliente en la empresa.Las preguntas que ayudan a entender el problema:¿Cuál es el rendimiento de los productos en las ventas?¿Cómo se relacionan los clientes con los productos en el tiempo?¿Cómo se agrupan los clientes según sus necesidades e intereses?¿Cómo retenemos a clientes o mejoramos las tasas de conversión a clientes ?  OBJETIVO:Este proyecto tiene como objetivo explorar diferentes herramientas de conversion, retención y rendimientos de clientes en las ventas a traves de metodologías de Machine LearningExploraremos y mediremos la efectividad de las siguientes herramientas:A. Product Analytics.B. Recomendación de ProductosC. CLV (Ciclo de vida del Valor cliente)D. Segmentación de clientes.Las técnicas algorítmicas en ML a utilizar y explorar:* No supervisados  * Clustering Knn* Supervidados  * Decision Tree,   * SVM, ANN, DNN  INICIALIZACION DE TOOLS<jupyter_code>import sys
assert sys.version_info >= (3, 5)
import os
%matplotlib inline
!pip install seaborn
!pip install kmodes
!pip install nltk
!pip install matplotlib
!pin install pydot<jupyter_output><empty_output><jupyter_text>TOOLS BASICS<jupyter_code>import pandas as pd
from pandas import DataFrame
%matplotlib inline
from datetime import datetime, timedelta, date
from pandas.plotting import autocorrelation_plot
from pandas import read_csv
from matplotlib import pyplot as plt


import warnings;
warnings.filterwarnings('ignore')


import seaborn as sns<jupyter_output><empty_output><jupyter_text>PRE PROCESAMIENTO<jupyter_code>from __future__ import division
from pandas import Series
from sklearn.metrics import classification_report,confusion_matrix
import datetime as dt
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>TOOLS DE NLP<jupyter_code># NLP
import re
import nltk 
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')
from nltk.corpus import stopwords 
from nltk.tokenize import word_tokenize, sent_tokenize
# ANALITICA DE TEXTO
from wordcloud import WordCloud
from wordcloud import STOPWORDS<jupyter_output><empty_output><jupyter_text>TOOLS ALGORITMO 1<jupyter_code>#Statistical LTV
from lifetimes import BetaGeoFitter, GammaGammaFitter
from lifetimes.utils import calibration_and_holdout_data, summary_data_from_transaction_data

from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import LinearSVC
from sklearn import preprocessing

from kmodes.kprototypes import KPrototypes
from pprint import pprint
import numpy as np
import plotly.graph_objects as go<jupyter_output><empty_output><jupyter_text>TOOLS ALGORITMO 2<jupyter_code># ML approach to LTV
import tensorflow as tf 
#import tensorflow_probability as tfp
from tensorflow import keras
from tensorflow.keras import layers
import tensorflow_docs as tfdocs
import tensorflow_docs.modeling as tfmodel
import tensorflow_docs.plots

# Ploteo 
import matplotlib
matplotlib.use('TkAgg')

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Evaluación
from sklearn.metrics import r2_score
from sklearn.metrics  import mean_absolute_error
# Clustering para clientes
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
print(f'TensorFlow Version: {tf.__version__}')
print(f'Keras version: {keras.__version__}')
print('Physical devices:')
tf.config.list_physical_devices()
%load_ext tensorboard
#remove previous fits
!rm -rf ./logs/ 

#set the directory to save fits:
import datetime
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")

#define the callback:
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
#start tensorboard:
%tensorboard --logdir logs/fit<jupyter_output><empty_output><jupyter_text>CARGA DE DATOS Si estas en Google Colab<jupyter_code>RETAIL_PATH = "https://github.com/hcgalvan/UNSAM-Machine-Learning-on-Economics/raw/main/data/"
if 'google.colab' in sys.modules:
  def load_datasets_h1(datasets_path=RETAIL_PATH):
    csv_path = os.path.join(datasets_path, "Year_2009-2010_train.csv")
    return pd.read_csv(csv_path, encoding= 'unicode_escape')

  retail_ol_h1 = load_datasets_h1()
  def load_datasets_h2(datasets_path=RETAIL_PATH):
      csv_path = os.path.join(datasets_path, "Year_2010-2011_train.csv")
      return pd.read_csv(csv_path, encoding= 'unicode_escape')

  retail_ol_h1 = load_datasets_h1()
  retail_ol_h2 = load_datasets_h2()<jupyter_output><empty_output><jupyter_text>Utilizar si estas en PC con Code y cualquier otro framework<jupyter_code>def carga_inicial():
    retail_ol_h1 = pd.read_csv('./data/Year_2009-2010_train.csv',  encoding= 'unicode_escape')
    retail_ol_h2 = pd.read_csv('./data/Year_2010-2011_train.csv',  encoding= 'unicode_escape')
    return [retail_ol_h1, retail_ol_h2]<jupyter_output><empty_output><jupyter_text>De Uso comun para PC<jupyter_code>def cargar_archivo_total():
    frames = carga_inicial()
    results = pd.concat(frames)
    df = results.copy()
    return df<jupyter_output><empty_output><jupyter_text>De Uso en COLAB<jupyter_code>def cargar_archivo_total_COLAB():
    results = pd.concat([retail_ol_h1, retail_ol_h2])
    df = results.copy()
    return df<jupyter_output><empty_output><jupyter_text>funcion lectura de un solo archivo 2010-2011<jupyter_code>def carga_archivo():
    frames = pd.read_csv('./data/Year_2010-2011_train.csv',  encoding= 'unicode_escape')
    df = frames.copy()
    return df<jupyter_output><empty_output><jupyter_text>EXPLORACION INICIAL DE DATOS  Situación actual de los datos<jupyter_code># cargar solo un archivo para pruebas
df = carga_archivo()
# cargar los dos archivos utilizados en TP
df = cargar_archivo_total()
df.shape
df.isnull().sum()
# Chequeamos datos unicos y actuales en cada features (atributos).

for i in df.columns:
  print("Actual number of values",i,len(df[i]))
  print("Unique number of values",i,len(df[i].unique()))
# Chequeamos valores nulos en los features
df.isnull().sum()
# Chequeamos datos duplicados
df.duplicated().sum()
# meses incompletos
print('Rango de Fecha: %s ~ %s' % (df['InvoiceDate'].min(), df['InvoiceDate'].max()))
df.loc[df['InvoiceDate'] >= '2011-12-01'].shape
df.loc[df['InvoiceDate'] < '2009-12-02' ].shape<jupyter_output>Rango de Fecha: 2010-12-01 08:26:00 ~ 2011-12-09 12:50:00<jupyter_text>PRE-PROCESAMIENTO DATOS  Preparación de Datos1. Eliminación de pedidos cancelados.2. Eliminando registros sin Customer ID, sin descripción de productos y features sin títulos3. Excluimos meses incompletos.4. Calcular las ventas totales de los features Cantidad y Precio unitario.5. Datos por cliente : para analizar segmentos de clientes, necesitamos transformar nuestros datos, de modo que cada registro represente el historial de compras de clientes individuales.  1. Limpieza de Datos<jupyter_code># Hay registros con valores negativos en la columna Cantidad, que representan pedidos cancelados. Ignoremos y eliminemos estos registros.
def limpieza_datos(df):
    # Observamos las cantidades negativas
    df.loc[df['Quantity'] <= 0].shape
    df = df.loc[df['Quantity'] > 0]
    #Quitamos la 1er columna vacía
    df.drop(['Unnamed: 0'], axis =1, inplace=True)
    # Quitamos valores nulos en features Customer ID y la Descripcion porque no son imputaciones.
    df.dropna(inplace=True)
    # Quitamos valores duplicados
    df.drop_duplicates(inplace=True)
    # Quitamos fechas incompletas
    df = df.loc[df['InvoiceDate'] < '2011-12-01']
    df = df.loc[df['InvoiceDate'] > '2009-12-01']
    return df
def limpieza_datos_con_cancel(df):
    # Quitamos valores nulos en features Customer ID y la Descripcion porque no son imputaciones.
    df.dropna(inplace=True)
    # Quitamos valores duplicados
    df.drop_duplicates(inplace=True)
    # Quitamos fechas incompletas
    df = df.loc[df['InvoiceDate'] < '2011-12-01']
    df = df.loc[df['InvoiceDate'] > '2009-12-01']
    return df<jupyter_output><empty_output><jupyter_text>2. Agregados de features  NLP - PRE-PROCESADO PARA ANALIZAR CATEGORIAS DE PRODUCTOS<jupyter_code># Esta función busca categorizar a los productos que se ofrecen
def agrega_color(df):
    colours = ['red','orange', 'yellow','green', 'blue', 'indigo', 'violet', 'purple', 'pink', 'silver', 'gold', 'beige', 'brown', 'grey', 'gray', 'black', 'white', 'cream']

    stop_words = set(stopwords.words('english'))
    Product_type = []
    Colour_type = []
    dataset= df
    # dataset= len(df)
    for row in dataset.iloc[:,2]:
        s=" "
        description = re.sub('[^a-zA-Z]'," ", str(row).lower()) #cleaning of text data
        wordsList = nltk.word_tokenize(description) #tokenization
        wordsList = [nltk.stem.WordNetLemmatizer().lemmatize(w, 'n') for w in wordsList if not w in stop_words] # lemmitization
        flag=False
        for w in wordsList:
            if w in colours:
                Colour_type.append(w)
                flag=True
            break
        if flag==False:
            Colour_type.append("no_color") #taking out colours from description

        tagged = nltk.pos_tag(wordsList)

        for tag in tagged:
            if tag[1]=='NN' :
                s+=tag[0] +  " "
        Product_type.append(s)
    
    return Product_type, Colour_type
def borrar_desc_invoice(df):
    # Quitar columnas "InvoiceDate" y "Description"
    X = df.drop(["Description", "InvoiceDate"], axis=1)
    return X
def cambiar_tipo_datos(df):
    """
    Transformar todas las variables en categoricas y en flotantes 
    Columna 2 es Quantity, 3 es Price y 8 Revenue
    """
    X = df.astype('category')
    X.iloc[:, 2] = X.iloc[:, 2].astype(float)
    X.iloc[:, 3] = X.iloc[:, 3].astype(float)
    X.iloc[:, 8] = X.iloc[:, 8].astype(float)
    return X<jupyter_output><empty_output><jupyter_text>FUNCIONES PARA DNN<jupyter_code>#  Ingeniería de features
def get_features(data, feature_start, feature_end, target_start, target_end):
    """
    Function that outputs the features and targets on the user-level.
    Inputs:
        * data - a dataframe with raw data
        * feature_start - a string start date of feature period
        * feature_end - a  string end date of feature period
        * target_start - a  string start date of target period
        * target_end - a  string end date of target period
    """
    features_data = data.loc[(data.date >= feature_start) & (data.date <= feature_end), :]
    print(f'Using data from {(pd.to_datetime(feature_end) - pd.to_datetime(feature_start)).days} days')
    print(f'To predict {(pd.to_datetime(target_end) - pd.to_datetime(target_start)).days} days')
    
    # Transacciones/Facturaciones
    total_rev = features_data.groupby('Customer ID')['Revenue'].sum().rename('total_revenue')
    recency = (features_data.groupby('Customer ID')['date'].max() - features_data.groupby('Customer ID')['date'].min()).apply(lambda x: x.days).rename('recency')
    frequency = features_data.groupby('Customer ID')['InvoiceDate'].count().rename('frequency')
    # la "t" es la fecha en mita del año
    t = features_data.groupby('Customer ID')['date'].min().apply(lambda x: (datetime(2011, 6, 11) - x).days).rename('t')
    time_between = (t / frequency).rename('time_between')
    
    avg_basket_value = (total_rev / frequency).rename('avg_basket_value')
    avg_basket_size = (features_data.groupby('Customer ID')['Quantity'].sum() / frequency).rename('avg_basket_Size')
    
    returns = features_data.loc[features_data['Revenue'] < 0, :].groupby('Customer ID')['InvoiceDate'].count().rename('num_returns')
    hour = features_data.groupby('Customer ID')['hour'].median().rename('purchase_hour_med')
    dow = features_data.groupby('Customer ID')['dayofweek'].median().rename('purchase_dow_med')
    weekend =  features_data.groupby('Customer ID')['weekend'].mean().rename('purchase_weekend_prop')
    
    #Datos para entrenamiento
    train_data = pd.DataFrame(index = rfm_train_test.index)
    train_data = train_data.join([total_rev, recency, frequency, t, time_between, avg_basket_value, avg_basket_size, returns, hour, dow, weekend])
    train_data = train_data.fillna(0)
    
    #Target Data
    target_data = data.loc[(data.date >= target_start) & (data.date <= target_end), :]
    target_quant = target_data.groupby(['Customer ID'])['date'].nunique()
    target_rev = target_data.groupby(['Customer ID'])['Revenue'].sum().rename('target_rev')
    # Cambios realizados, joint con target_quant HUG
    train_data = train_data.join(target_rev, target_quant).fillna(0) 
    
    return train_data.iloc[:, :-1], train_data.iloc[:, -1]
#DNN - Red Neuronal Profunda -> Regularizador haciendo con apagado de capas
def build_model():
    
    model = keras.Sequential(layers.ImputLayer(imput_shape=(1,), name='Input'),layers=[
    layers.Dense(256, activation='relu', input_shape=[len(X_train.columns), ]),
    layers.Dropout(rate=0.3),
    layers.Dense(64, activation='relu'),
    layers.Dropout(rate=0.3),
    layers.Dense(32, activation='relu'),
    layers.Dense(1)
    ])


    optimizer = tf.keras.optimizers.Adam(lr=0.001)
   
    model.compile(loss='mse',
            optimizer=optimizer,
            metrics=['mae', 'mse'])
    
    return model
    
# Evaluación de predicción
def evaluate(actual, sales_prediction):
    print(f"Total Ventas Actual: {np.round(actual.sum())}")
    print(f"Total Ventas Predictivo: {np.round(sales_prediction.sum())}")
    print(f"Individual R2 score: {r2_score(actual, sales_prediction)} ")
    print(f"Individual Mean Absolute Error: {mean_absolute_error(actual, sales_prediction)}")
    plt.scatter(sales_prediction, actual)
    plt.xlabel('Prediction')
    plt.ylabel('Actual')      
    plt.show()<jupyter_output><empty_output><jupyter_text>FUNCIONES UTILIZADAS EN GENERAL<jupyter_code># Damos formato de fecha a InvoiceDate para realizar tratamientos posteriores
def tipos_dataset(df):    
    df['InvoiceDate']  = pd.to_datetime(df.InvoiceDate, format = '%Y/%m/%d %H:%M')
    df['Quantity'] = pd.to_numeric(df['Quantity'], errors='coerce')
    df['Price']=df['Price'].astype(str)
    df['Price']=df['Price'].astype(float)
    df['Month'] = pd.DatetimeIndex(pd.to_datetime(df['InvoiceDate'])).month
    return df
# Agregamos features para Analizar diferentes casos
# UTILIZAR UNA VEZ SETEADO LOS TIPOS DE DATOS def tipos_dataset(df)
def agregados_features(df):
    # Agregamos features Sales
    df['Sales'] = df['Quantity'] * df['Price']
    # Datos cliente por pedido
    df['date'] = pd.to_datetime(df.InvoiceDate.dt.date, errors='coerce')
    df['time'] = df.InvoiceDate.dt.time
    df['hour'] = df['time'].apply(lambda x: x.hour)

    df['weekend'] = df['date'].apply(lambda x: x.weekday() in [5, 6])
    df['dayofweek'] = df['date'].apply(lambda x: x.dayofweek)

    df['Product Type'] = agrega_color(df)[0]
    df['Colour_type']= agrega_color(df)[1]

    return df
def agregar_fecha_log2(df):
    #Get revenue column
    df['Revenue'] = df['Quantity'] * df['Price']
    #Datetime transformation
    df['date'] = pd.to_datetime(df.InvoiceDate.dt.date)
    df['time'] = df.InvoiceDate.dt.time
    df['hour'] = df['time'].apply(lambda x: x.hour)
    df['weekend'] = df['date'].apply(lambda x: x.weekday() in [5, 6])
    df['dayofweek'] = df['date'].apply(lambda x: x.dayofweek)
    return df
def agregar_fechas(df):
 #Datetime transformation
    df['InvoiceDate']  = pd.to_datetime(df.InvoiceDate, format = '%Y/%m/%d %H:%M')
    # df['date'] = pd.to_datetime(df.InvoiceDate.dt.date)
    df['Day'] = df['InvoiceDate'].apply(lambda x: x.day)
    df['Month'] = df['InvoiceDate'].apply(lambda x: x.month)
    df['Year'] = df['InvoiceDate'].apply(lambda x: x.year)
    df['DayOfWeek'] = df['InvoiceDate'].apply(lambda x: x.dayofweek)
    return df<jupyter_output><empty_output><jupyter_text>FUNCIONES SOBRE DESPCRIPCION DE CLIENTES<jupyter_code>def graficar_cluster(four_cluster_df):
    plt.scatter(
    four_cluster_df.loc[four_cluster_df['Cluster'] == 0]['OrderCount'], 
    four_cluster_df.loc[four_cluster_df['Cluster'] == 0]['TotalSales'],
    c='blue'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 1]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 1]['TotalSales'],
        c='red'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 2]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 2]['TotalSales'],
        c='orange'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 3]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 3]['TotalSales'],
        c='green'
    )

    plt.title('TotalSales vs. OrderCount Clusters')
    plt.xlabel('Order Count')
    plt.ylabel('Total Sales')

    plt.grid()
    plt.show()


    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 0]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 0]['AvgOrderValue'],
        c='blue'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 1]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 1]['AvgOrderValue'],
        c='red'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 2]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 2]['AvgOrderValue'],
        c='orange'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 3]['OrderCount'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 3]['AvgOrderValue'],
        c='green'
    )

    plt.title('AvgOrderValue vs. OrderCount Clusters')
    plt.xlabel('Order Count')
    plt.ylabel('Avg Order Value')

    plt.grid()
    plt.show()


    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 0]['TotalSales'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 0]['AvgOrderValue'],
        c='blue'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 1]['TotalSales'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 1]['AvgOrderValue'],
        c='red'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 2]['TotalSales'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 2]['AvgOrderValue'],
        c='orange'
    )

    plt.scatter(
        four_cluster_df.loc[four_cluster_df['Cluster'] == 3]['TotalSales'], 
        four_cluster_df.loc[four_cluster_df['Cluster'] == 3]['AvgOrderValue'],
        c='green'
    )

    plt.title('AvgOrderValue vs. TotalSales Clusters')
    plt.xlabel('Total Sales')
    plt.ylabel('Avg Order Value')

    plt.grid()
    plt.show()
    return<jupyter_output><empty_output><jupyter_text>FUNCIONES SOBRE DESPCRIPCION DE PRODUCTOS<jupyter_code># Ploteo de letras
def ploteo_word(df):
    stopwords=set(STOPWORDS)
    wordcloud=WordCloud(background_color='White').generate(str(df['Description']))
    print(wordcloud)
    plt.rcParams['figure.figsize']=(12,12)
    plt.axis('off')
    plt.imshow(wordcloud)
    plt.title('Lista de Cantidad veces utilizada una letra descripción productos',fontsize=20,color='red')
    plt.show()
    return<jupyter_output><empty_output><jupyter_text>FUNCIONES SOBRE PRECIOS<jupyter_code># Ploteo Cantidad y precio
def ploteo_precio_cantidad(df):
    df = df[['Quantity','Price']]
    df['PriceBins'] = pd.cut(df['Price'].tolist(), bins=8)
    sns.barplot(data=df,x="PrecioBins", y="Cantidad")
    return
#Cuántos clientes compraron algo cada mes durante el último año
def precio_cliente_pais(df):
    
    df = df.groupby(['Country' , 'Month']).agg({'Price':'sum' , 'Customer ID' :'count'})
    df.columns = ['PriceSum','CustomerIDCount']
    df = df.reset_index()
    cm = sns.light_palette("blue", as_cmap=True)
    pvd = pd.pivot_table(df, values='CustomerIDCount', index=['Country'],
                    columns=['Month'],
                    aggfunc=np.sum).fillna(0)
    return pvd.style.background_gradient(cmap=cm)
# ¿En qué rangos los precios son más comunes? - Grafica
def rango_precios(df):
    prices = pd.DataFrame([df['Price'].value_counts()
                         .sort_values(ascending=False).to_dict()]).T
    prices = pd.DataFrame(prices['Price'].value_counts())
    prices = prices.reset_index()
    prices.columns = ['Price','CountPrice']
    prices['PriceBins'] = prices.cut(df['Price'].tolist(), bins=8)
    sns.barplot(data=prices, x='PriceBins', y='CountPrice')
    return
# Rango de cambio de precio durante el tiempo - EXPLORACION DE DATOS
def AED(df):
     customer_avg_spending= df[['Price','Customer ID', 'InvoiceDate' , 'Country']]
     avg_selling_of_products = df[['Price','Quantity','InvoiceDate']]
     return customer_avg_spending, avg_selling_of_products 
def customer_avg_spending_insights(df):
    df['InvoiceDate'] = pd.to_datetime(df['InvoiceDate']).dt.strftime('%Y-%m-%d')
    df = df.groupby(['InvoiceDate']).agg({'Price':'sum'}).reset_index()
    df = df.reset_index(drop=True)
    df.columns = ['Date','PriceSum']
    fig = go.Figure()
    fig.add_trace(go.Scatter(x=df['Date'], y=df['PriceSum'], name="Price Sum",
                         line_color='deepskyblue'))
    fig.update_layout(title_text='Sum range of all prices among time',
                  xaxis_rangeslider_visible=True)
    fig.show()
    return
# Esta función agrupa por continente a paises y suma la cantidad vendida por pais
def cantidad_mensual_pais(df):
    df = df.groupby(['Country' , 'Month']).agg({'Quantity':'sum'})
    df = df.reset_index()
    df = df.sort_values(by=['Month'])
    Europe = ['United Kingdom','France', 'Belgium','EIRE',
              'Germany','Portugal', 'Denmark', 'Netherlands', 'Poland',
             'Spain', 'Cyprus', 'Greece', 'Norway', 'Austria', 'Sweden', 
              'Finland','Italy', 'Switzerland', 'Malta', 'Israel', 
              'Lithuania','Iceland']
    Asia = [ 'Japan','United Arab Emirates','Singapore','Hong Kong',
       'Thailand','West Indies', 'Korea','Lebanon',]
    America = ['Channel Islands','USA','Brazil', 'Canada']
    Australia = ['Australia',]
    df['Continent'] = df['Country'].map(lambda x: 'Europe' if x in Europe else(
                                        'Asia' if x in Asia else
                                        'America' if x in America else
                                        'Australia' if x in Australia else 'None' ))
    fig = px.scatter_geo(df, locations="Country",color="Continent",
                         hover_name="Country", size="Quantity",
                         animation_frame="Month",
                         projection="natural earth")
    fig.show()
    return
def create_prediction(df):
    topic = df['Topic']
    df = df[['InvoiceDate','Price']]
    df.columns = ['ds','y']
    m = Prophet()
    m.fit(df)
    future = m.make_future_dataframe(periods=90)
    forecast = m.predict(future)
    df_cv = cross_validation(m, horizon='90 days')
    df_p = performance_metrics(df_cv)
    fig3 = plot_cross_validation_metric(df_cv, metric='mape')
    return m.plot(forecast) ,  m.plot_components(forecast) , df_p , fig3<jupyter_output><empty_output><jupyter_text>FUNCIONES QUE HACEN AGREGACIONES Y MATRICES<jupyter_code>def matriz_cliente_item(df):
    # Matriz cliente-item
    # Podemos sumar todas las cantidades compradas para cada artículo, utilizando la función aggfunc.
    customer_item_matrix = df.pivot_table(
        index='Customer ID', 
        columns='StockCode', 
        values='Quantity',
        aggfunc='sum'
    )
    #Convertimos esta matriz y lo codificamos en 0 - 1 a los datos, por lo que el valor de 1 determinando un producto fue comprado por el cliente dado, y el valor 0 determinado por producto que nunca fue comprado por el cliente dado.
    # La función Lambda que estamos usando en este código simplemente codifica todos los elementos cuyos valores son mayores que 0 con 1, y el resto con 0.
    customer_item_matrix = customer_item_matrix.applymap(lambda x: 1 if x > 0 else 0)
    return customer_item_matrix
# Cantidad de pedidos por cliente
def agrupa_ordenesxcliente(df):
    orders_df = df.groupby(['Customer ID', 'Invoice']).agg({
    'Sales': sum,
    'InvoiceDate': max
    })
    return orders_df
# Ventas por cliente
def agrupa_ventasxcliente(df):
    customer_df = df.groupby('Customer ID').agg({
        'Sales': sum,
        'Invoice': lambda x: x.nunique()
    })

    customer_df.columns = ['TotalSales', 'OrderCount']
    customer_df['AvgOrderValue'] = customer_df['TotalSales']/customer_df['OrderCount']
    return customer_df
# Cantidad de Artículos por Orden/Factura
def agrupa_artxventas(df):
    group_by_invoice=pd.DataFrame(df.groupby('Invoice')['StockCode'].nunique())
    group_by_invoice.columns=['No.of Items per Order']
    group_by_invoice.head()
    return<jupyter_output><empty_output><jupyter_text>ANÁLISIS: CARACTERIZACIÓN DE DATOS Necesitamos convertir InvoiceDate en tipo Date.<jupyter_code>df.describe(include='object')<jupyter_output><empty_output><jupyter_text>PRE-PROCESAMIENTO PARA CLIENTES  A partir de 4-CustomerSegmentation.ipynb  Limpieza inicial y agregados<jupyter_code>df = limpieza_datos(df)
df = tipos_dataset(df)
df['Sales'] = df['Quantity'] * df['Price']<jupyter_output><empty_output><jupyter_text>Caracterización de clientes<jupyter_code># Analizamos clientes
df['Customer ID'].describe()
df['Price'].describe()
### Visualizacion de los cuantiles
plt.hist(df['Price'], bins = 50, range = [1.2, 2.4], density = True, cumulative = True, rwidth = 0.80)

# Cambiar aqui lo cuantiles que se quieran visualizar
for c in [0.20, 0.80]:

  plt.hlines(y = c, xmin = 1.2, xmax = np.quantile(df['Price'], c), linestyle = '--', color = 'r', alpha = 0.65, linewidth = 3)
  plt.vlines(x = np.quantile(df['Price'], c), ymin = 0.00, ymax = c, linestyles= '--', color = 'r', alpha = 0.65, linewidth = 3)

plt.ylabel('Probabilidad acumulada')
plt.ylim([0.00, 1.05])
plt.xlim([1.2, 2.4])
plt.show()
Precio = df['Price']

# Histograma
ax = Precio.hist(bins = 1, grid = False, rwidth = 0.80, cumulative = False, density = True)
ax.set_xlabel('Cantidad')
ax.set_ylabel('Frecuencia')
plt.show()

# Observables
print('Media: {:3f}'.format(Precio.mean()))
print('Varianza: {:.3f}'.format(Precio.var()))
print('Desviacion: {:.3f}'.format(Precio.std()))
print('90% central: {} - {}'.format(*Precio.quantile([0.05, 0.95])))
print('Mediana: {}'.format(Precio.median()))
print('Moda: {}'.format(Precio.mode().values))
df['Quantity'].describe()
clientes = agrupa_ventasxcliente(df)
clientes.describe()
# Rankeo de clientes
rank_df = clientes.rank(method='first')
rank_df.describe()
# Ranking de clientes
rank_df.head(15)<jupyter_output><empty_output><jupyter_text>Normalización de datos<jupyter_code># Normalizamos el ranking de clientes
normalized_df = (rank_df - rank_df.mean()) / rank_df.std()
normalized_df.head(15)
normalized_df.describe()<jupyter_output><empty_output><jupyter_text>PRE-PROCESAMIENTO PARA CLIENTES A PARTIR DE PRODUCTOS  Limpieza inicial y agregadosA partir del Test_OnlineRetail.ipynb en estos momentos el analisis lo realizo directamente desde TP-EcoDat-Part-II-Testonlineretaial.ipynb<jupyter_code># cargar solo un archivo para pruebas
df1 = carga_archivo()

nuevodf = limpieza_datos_con_cancel(df1)
# ¿Cuál es el produto más vendido?
nuevodf.StockCode.mode()
product = nuevodf[nuevodf.StockCode.str.contains("85123A")]
product.head()
nuevodf.Description.mode()
# ¿Cuántos clientes tienen compras para más de un (1) país?
customer= df1.groupby("Customer ID")["Country"].count()>1
customer.count()

# ¿Qué países tienen el mayor y menor número de cancelaciones?
cancelled_orders = df1[df1.Invoice.str.contains("C")]
cancelled_orders.head()
cancelled_orders.Country.mode()
cancelled_orders.Country.min()
# ¿En qué momento se realizan más compras?
data = nuevodf.InvoiceDate.str.split(expand = True)
data.columns = ['Date', 'Time']
data
nuevodf.duplicated().sum()
# En promedio, ¿cuántos artículos se compran por factura?
avg = df1.groupby('Invoice')['Quantity'].sum().reset_index()
avg.head()<jupyter_output><empty_output><jupyter_text>PRE-PROCESAMIENTO PARA Segmentación de clientes  Clusterizacion con kmeans<jupyter_code>kmeans = KMeans(n_clusters=4).fit(normalized_df[['TotalSales', 'OrderCount', 'AvgOrderValue']])
kmeans
kmeans.labels_
# Centros de cada grupo
kmeans.cluster_centers_
four_cluster_df = normalized_df[['TotalSales', 'OrderCount', 'AvgOrderValue']].copy(deep=True)
four_cluster_df['Cluster'] = kmeans.labels_
four_cluster_df.head()

four_cluster_df.groupby('Cluster').count()['TotalSales']
graficar_cluster(four_cluster_df)<jupyter_output><empty_output><jupyter_text>Seleccionar el mejor numero de cluster<jupyter_code># Creamos una lista para guardar de los coeficientes de silhouette para cada valor de k
silhouette_coefficients = []

for n_cluster in range(2, 8):
    kmeans = KMeans(n_clusters=n_cluster)
    kmeans.fit(
        normalized_df[['TotalSales', 'OrderCount', 'AvgOrderValue']]
    )
    silhouette_avg = silhouette_score(
        normalized_df[['TotalSales', 'OrderCount', 'AvgOrderValue']], 
        kmeans.labels_
    )
    silhouette_coefficients.append(silhouette_avg)
    
    print('Silhouette Score for %i Clusters: %0.4f' % (n_cluster, silhouette_avg))
fig, ax = plt.subplots(figsize = (24, 7))

# estas lineas son el grafico de SSEvsK
ax.scatter(range(2, 8), silhouette_coefficients)            
ax.set_xticks(range(2, 8))
ax.set_xlabel("Número de clusters")
ax.set_ylabel("Promedio coeficientes de Silhouette")
import numpy as np<jupyter_output><empty_output><jupyter_text>Interpretación de la segmentación de clientes<jupyter_code>kmeans = KMeans(n_clusters=4).fit(
    normalized_df[['TotalSales', 'OrderCount', 'AvgOrderValue']]
)
four_cluster_df = normalized_df[['TotalSales', 'OrderCount', 'AvgOrderValue']].copy(deep=True)
four_cluster_df['Cluster'] = kmeans.labels_
kklabels = kmeans.labels_
kkklabels = kklabels.copy()

np.random.shuffle(kkklabels)

silhouette_score(four_cluster_df , kkklabels)
four_cluster_df.head(15)
kmeans.cluster_centers_
high_value_cluster = four_cluster_df.loc[four_cluster_df['Cluster'] == 3]
high_value_cluster.head()
clientes.loc[high_value_cluster.index].describe()
pd.DataFrame(
    df.loc[
        df['Customer ID'].isin(high_value_cluster.index)
    ].groupby('Description').count()[
        'StockCode'
    ].sort_values(ascending=False).head()
)<jupyter_output><empty_output><jupyter_text>Análisis Agrupados<jupyter_code># ¿Cual es la facturación por cliente?
vtaxcliente = agrupa_ventasxcliente(df)
vtaxcliente

#Caracterización de estos datos
vtaxcliente.describe()
#¿Cuántas Ordenes/Facturas se realizaron por cliente?
ordenes = agrupa_ordenesxcliente(df)
ordenes

# Caracterización de estos datos
ordenes.describe()
# ¿Cuántos artículos se incluyen en cada Orden/Factura?
artxcliente = agrupa_artxventas(df)



artxcliente.columns()
# Caracterización de estos datos
artxcliente.describe()
# ¿Cuál es la media por artículos por cada cliente?

# Caracterización de estos datos<jupyter_output><empty_output><jupyter_text>Análisis de Precios y Cantidades<jupyter_code>df.info()
df1 = tipos_dataset(df)<jupyter_output><empty_output><jupyter_text>ALGORITMOS ANALIZADOS  ALGORITMO 1 - CLASIFICACION Y PREDICCIÓN DE CANTIDAD (RandomForestClassifier, KNN, SVC with kernel, AdaBoost, logistic, Naive base Classifier, Decision Tree Classifier, GradientBoostingClassifier )<jupyter_code>df = cargar_archivo_total_COLAB()
# Limpiar datos y establecer tipos
df = limpieza_datos(df)
# Agregar columnas en Dataframe
df['Product Type'] = agrega_color(df)[0]

df['Colour_type'] = agrega_color(df)[1]


df<jupyter_output><empty_output><jupyter_text>Cargar archivo df LEVANTAR EN GOOGLE COLAB<jupyter_code>RETAIL_PATH = "https://github.com/hcgalvan/UNSAM-Machine-Learning-on-Economics/raw/main/data/"
if 'google.colab' in sys.modules:
  def load_datasets_df(datasets_path=RETAIL_PATH):
    csv_path = os.path.join(datasets_path, "df.csv")
    return pd.read_csv(csv_path, encoding= 'unicode_escape')
df = load_datasets_df()
# Borrar Descripcion e Invoice Date
df1 = borrar_desc_invoice(df)

# Agregar Ingresos por Ventas
df1['Revenue'] = df1['Price'] * df1['Quantity']
df1.head()
# Codificación de Label para features categoricos
# LabelEncoderes es una clase de utilidad para ayudar a normalizar etiquetas 
# de modo que solo contengan valores entre 0 y n_classes-1
label_encoder = preprocessing.LabelEncoder()

for col in ["Invoice", "StockCode", "Customer ID","Country", "Product Type","Colour_type"]:
  df1[col] = label_encoder.fit_transform(df1[col])
df1.head()
# Definir los tipos de datos en categoricos y Float
df1 = cambiar_tipo_datos(df1)

# Splitear en dataframe en "Train" y "test"
train, test = train_test_split(df1, train_size = 0.8, random_state = 0)<jupyter_output><empty_output><jupyter_text>Cluster de items similares para el nuevo feature cluster, utiliza en este caso K-prototype clustering<jupyter_code># Chequeo del valor optimo de 'K' // demanda varios minutos de ejecucion para encontrar el optimo
# Luego de 2067 minutos me arroja un error, ""
# "ValueError: Clustering algorithm could not initialize. Consider assigning the initial clusters manually."
# Las lecturas en https://github.com/nicodv/kmodes/blob/master/README.rst#faq me indica que es una caracteristica 
# donde no tiene sentido
# a.Ejecutar con menos clústeres, ya que es posible que los datos no admitan una gran cantidad de clústeres.
# b.Explore y visualice sus datos, verificando distribuciones extrañas, valores atípicos, etc.
# c.Limpiar y normalizar los datos
# d.Aumentar la proporción de filas a columnas. 

#cost = []
#for num_clusters in list(range(1,15)):
#    kproto = KPrototypes(n_clusters = num_clusters, init='Cao')
#    kproto.fit_predict(train, categorical=[0, 1, 4, 5, 6, 7])
#    cost.append(kproto.cost_)
#    labels=kproto.labels_
#plt.plot(cost)


# a partir de la lectura https://medium.com/@shailja.nitp2013/k-modesclustering-ef6d9ef06449
# k-modes se utiliza para agrupar variables categóricas. Define agrupaciones en función del número de categorías coincidentes entre puntos de datos.
# (Esto contrasta con el algoritmo k-medias más conocido, que agrupa datos numéricos basados ​​en la distancia euclidiana). 
# El algoritmo k-prototipos combina k-modos y k-medias y es capaz de agrupar datos numéricos / categóricos mixtos.
cost = []
for num_clusters in list(range(2,5)): # 2, 3 y 4
    kproto = KPrototypes(n_clusters = num_clusters, max_iter=23, init='Cao', n_init=1, verbose=1)
    kproto.fit_predict(train, categorical=[0, 1, 4, 5, 6, 7])
    cost.append(kproto.cost_)
    labels=kproto.labels_
plt.plot(cost)
#cost = []
#for num_clusters in list(range(1,5)):
#    kmode = KModes(n_clusters=num_clusters, init = "Cao", n_init = 1, verbose=1)
#    kmode.fit_predict(df)
#    cost.append(kmode.cost_)
#    y = np.array([i for i in range(1,5,1)])
#plt.plot(y,cost)
# Genera un nuevo numero de cluster del atributo
# , max_iter=10, init='Cao', n_init=1, verbose=1
kproto = KPrototypes(n_clusters = 3, init = 'Cao', n_init=1, verbose=1)
kproto.fit_predict(train, categorical=[0, 1, 4, 5, 6])
print(kproto.cost_)
labels=kproto.labels_
# Agrego nuevo atributo
train["Cluster number"]=labels

# Ahora agrego "InvoiceDate" en dataframe

df2 = train.merge(pd.DataFrame(df["InvoiceDate"]), left_index=True, right_index=True)<jupyter_output><empty_output><jupyter_text>Ingeniería de feature<jupyter_code>df2
# Agrego las fechas en diferentes rangos: horas, dias, meses, años
df2 = agregar_fechas( df2 )
df2<jupyter_output><empty_output><jupyter_text>GUARDAR Y LUEGO LEVANTAR EN PC<jupyter_code># Guardo el proceso en archivo csv
df2.to_csv('./data/df2.csv')
# SI FUNCIONA PARA DESCARGAR Y LUEGO LO LEVANTO  PARA TRABAJAR CON GOOGLECOLAB
from google.colab import files
df2.to_csv('df2.csv') 
files.download('df2.csv')
# Capturo el archivo, quito columna vacía y muestro
df2 = pd.read_csv('./data/df2.csv')
df2.drop(['Unnamed: 0'], axis =1, inplace=True)
df2.drop(['InvoiceDate'], axis=1, inplace=True)
df2.head()<jupyter_output><empty_output><jupyter_text>LEVANTAR EN GOOGLE COLAB<jupyter_code>RETAIL_PATH = "https://github.com/hcgalvan/UNSAM-Machine-Learning-on-Economics/raw/main/data/"
if 'google.colab' in sys.modules:
  def load_datasets_df2(datasets_path=RETAIL_PATH):
    csv_path = os.path.join(datasets_path, "df2.csv")
    return pd.read_csv(csv_path, encoding= 'unicode_escape')
df2 = load_datasets_df2()
df2.drop(['Unnamed: 0'], axis =1, inplace=True)
df2.drop(['InvoiceDate'], axis=1, inplace=True)
df2.head()<jupyter_output><empty_output><jupyter_text>Classification of test data into number of clusters- Cluster numbers were treated as a target variable as the objectivewas to match the records from the validation and testing sets with the clusters from the training set.- El numero de Clusters fueron tratados como variable target, con el objetivo de converger los registros del set de validación y testeo con el cluster del set de entrenamiento.<jupyter_code># Corte del dataframe entre entrenamiento y validación
train_, val_= train_test_split(df2, train_size = 0.8, random_state = 0)
train_y=train_["Cluster number"]
train_x=train_.drop(['Cluster number'],axis=1,inplace=False)

val_y=val_["Cluster number"]
val_x=val_.drop(['Cluster number'],axis=1,inplace=False)<jupyter_output><empty_output><jupyter_text>Para Clasificar* Utilizamos SVC, que brinda el mejor resultado sobre otros algoritmos<jupyter_code>model1 = LinearSVC(verbose=True)
model1.fit(train_x, train_y)<jupyter_output>[LibLinear]<jupyter_text>Graficador<jupyter_code># Representación gráfica de los límites de clasificación
# ==============================================================================
# Grid de valores
x = np.linspace(np.min(train_.val_x), np.max(train_.val_x), 50)
y = np.linspace(np.min(train_.val_y), np.max(train_.val_y), 50)
Y, X = np.meshgrid(y, x)
grid = np.vstack([X.ravel(), Y.ravel()]).T

# Predicción valores grid
pred_grid = model1.predict(grid)

fig, ax = plt.subplots(figsize=(6,4))
ax.scatter(grid[:,0], grid[:,1], c=pred_grid, alpha = 0.2)
ax.scatter(train_x.val_x, train_x.val_y, c=train_y, alpha = 1)

# Vectores soporte
ax.scatter(
    model1.support_vectors_[:, 0],
    model1.support_vectors_[:, 1],
    s=200, linewidth=1,
    facecolors='none', edgecolors='black'
)

# Hiperplano de separación
ax.contour(
    X,
    Y,
    model1.decision_function(grid).reshape(X.shape),
    colors = 'k',
    levels = [-1, 0, 1],
    alpha  = 0.5,
    linestyles = ['--', '-', '--']
)

ax.set_title("Resultados clasificación SVM lineal");<jupyter_output><empty_output><jupyter_text>Procesos<jupyter_code># Validación del data test
pred_y = model1.predict(val_x)
# Evaluación de Performance
accuracy_score(val_y,pred_y)
# Adding "InvoiceDate" in test data

test_Df = test.merge(pd.DataFrame(df["InvoiceDate"]), left_index=True, right_index=True)
test_Df.head()
# Probamos ahora con el test
test_Df = agregar_fechas( test_Df )
test_Df.drop(['InvoiceDate'],axis=1, inplace=True)
test_Df=test_Df.reset_index()
test_Df.drop(["index"],axis=1, inplace = True)
test_Df.head()
# Predecimos el numero de Cluster en datos de testeo
test_Df['Cluster number'] = model1.predict(test_Df)
test_Df<jupyter_output><empty_output><jupyter_text>PREDICCION DE FEATURE "QUANTITY" PARA DEMANDAS DE PRODUCTOS<jupyter_code>train_y=train_["Quantity"].astype('int')
train_x=train_.drop(['Quantity'], axis=1,inplace=False)

test_Df_y=test_Df["Quantity"].astype('int')
test_Df_x=test_Df.drop(['Quantity'],axis=1,inplace=False)
train_x.columns<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(bootstrap=True,ccp_alpha=0.0,
                                                   max_depth=None,
                                                   max_features='auto',
                                                   max_leaf_nodes=None,
                                                   max_samples=None,
                                                   min_impurity_decrease=0.0,
                                                   #min_impurity_split=None,
                                                   min_samples_leaf=1,
                                                   min_samples_split=2,
                                                   min_weight_fraction_leaf=0.0,
                                                   n_estimators=100,
                                                   n_jobs=None,)
# Entrenamos
clf.fit(train_x, train_y)
#Function to visualize tree
def visualize_tree(data,clf,clf_name):
	features = data.columns
	features = features[:-1]
	class_names = list(set(data.iloc[:,-1]))
	dot_data = export_graphviz(clf, out_file=None,  \
		feature_names=features,class_names=class_names,  \
		filled=True, rounded=True, special_characters=True)
	graph = graphviz.Source(dot_data)
	graph.render('dtree_render_'+clf_name,view=True)
visualize_tree(train_, clf, 'test' )
# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os
import sys

import matplotlib.pyplot as plt
%matplotlib inline

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)
# Let us define a couple of useful functions (if in colab, otherwise, take from utils module)
if 'google.colab' in sys.modules:

    def plot_clasi(x, t, ws, labels=[], xp=[-1., 1.], thr=[0,], spines='zero', equal=True, 
                   join_centers=False, margin=None):
        """
        Figura con el resultado del ajuste lineal
        """
        assert len(labels) == len(ws) or len(labels) == 0
        assert len(ws) == len(thr)

        if margin is None:
            margin = [False] * len(ws)
        else:    
            margin = np.atleast_1d(margin)
        assert len(margin) == len(ws)

        if len(labels) == 0:
            labels = np.arange(len(ws)).astype('str')

        # Agregemos el vector al plot
        fig = plt.figure(figsize=(9, 7))
        ax = fig.add_subplot(111)

        xc1 = x[t == np.unique(t).max()]
        xc2 = x[t == np.unique(t).min()]

        ax.plot(*xc1.T, 'ob', mfc='None', label='C1')
        ax.plot(*xc2.T, 'or', mfc='None', label='C2')

        for i, w in enumerate(ws):

            # Compute vector norm
            wnorm = np.sqrt(np.sum(w**2))

            # Ploteo vector de pesos
            x0 = 0.5 * (xp[0] + xp[1])
            ax.quiver(0, thr[i]/w[1], w[0]/wnorm, w[1]/wnorm, 
                      color='C{}'.format(i+2), scale=10, label=labels[i], 
                      zorder=10)

            # ploteo plano perpendicular
            xp = np.array(xp)
            yp = (thr[i] - w[0]*xp)/w[1] 

            plt.plot(xp, yp, '-', color='C{}'.format(i+2))

            # Plot margin
            if margin[i]:
                for marg in [-1, 1]:
                    ym = yp + marg/w[1]
                    plt.plot(xp, ym, ':', color='C{}'.format(i+2))


        if join_centers:
            # Ploteo línea que une centros de los conjuntos
            mu1 = xc1.mean(axis=1)
            mu2 = xc2.mean(axis=1)
            ax.plot([mu1[0], mu2[0]], [mu1[1], mu2[1]], 'o:k', mfc='None', ms=10)    

        ax.legend(loc=0, fontsize=12)
        if equal:
            ax.set_aspect('equal')

        if spines is not None:
            for a in ['left', 'bottom']:
                ax.spines[a].set_position('zero')
            for a in ['top', 'right']:
                ax.spines[a].set_visible(False)

        return


    def makew(fitter):

        # # Obtengamos los pesos y normalicemos
        w = fitter.coef_.copy()

        # # Incluye intercept
        if fitter.fit_intercept:
            w = np.hstack([fitter.intercept_.reshape(1,1), w])

        # # Normalizon
        #w /= np.linalg.norm(w)
        return w.T
else:
  from Utils import plot_clasi
#Utility form <NAME>
def plot_decision_regions(clf, X, t, axes=None, npointsgrid=500, legend=False, 
                          plot_training=True, 
                          figkwargs={'figsize': [12, 8]}, 
                          contourkwargs={'alpha':0.3}):
    """
    Plot decision regions produced by classifier.

    :param Classifier clf: sklearn classifier supporting XXX
    """

    fig = plt.figure(**figkwargs)
    ax = fig.add_subplot(111)

    if axes is None:
        dx = X[:, 0].max() - X[:, 0].min()
        dy = X[:, 1].max() - X[:, 1].min()
        axes = [X[:, 0].min() - 0.1*dx, X[:, 0].max() + 0.1*dx, 
                X[:, 1].min() - 0.1*dy, X[:, 1].max() + 0.1*dy]
        
    # Define grid for regions
    x1s = np.linspace(axes[0], axes[1], npointsgrid)
    x2s = np.linspace(axes[2], axes[3], npointsgrid)
    x1, x2 = np.meshgrid(x1s, x2s)

    # Make predictions on points of grid; reshape to grid format
    X_new = np.c_[x1.ravel(), x2.ravel()]
    y_pred = clf.predict(X_new).reshape(x1.shape)

    #custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
    ax.contourf(x1, x2, y_pred, **contourkwargs)

#     custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
#         plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)

    if plot_training:
        for label in np.unique(t):
            ax.plot(X[:, 0][t==label], X[:, 1][t==label], "o", label="C{}".format(label))

    # Axis
    plt.xlabel(r"$x_1$", fontsize=18)
    plt.ylabel(r"$x_2$", fontsize=18, rotation=0)

    if legend:
        plt.legend(loc="lower right", fontsize=14)

    plt.show()
    return fig
!pip install graphviz
from sklearn.tree import export_graphviz
import cv2

from sklearn.tree import export_graphviz
estimator = clf.estimators_[5]
features = train_x.columns
#class_names = list(set(train_x.iloc[:,-1]))
dot_data = export_graphviz(estimator, out_file=None,  \
		feature_names=features,class_names=None,  \
		filled=True, rounded=True, special_characters=True)
graph = graphviz.Source(dot_data)
graph.render(view=True)
def visualize_tree(data,clf,clf_name):
	features = data.columns
	features = features[:-1]
	class_names = list(set(data.iloc[:,-1]))
	dot_data = tree.export_graphviz(clf, out_file=None,  \
		feature_names=features,class_names=["Gana visitante","Empate","Gana local"],  \
		filled=True, rounded=True, special_characters=True)
	graph = graphviz.Source(dot_data)
	graph.render('dtree_render_'+clf_name,view=True)
estimator = clf.estimators_[5]
export_graphviz(
    estimator,
    out_file="tree.dot",
    feature_names=train_x.columns,
    #class_names=["Gana visitante","Empate","Gana local"],
    rounded=True,
    filled=True
)
import graphviz
from sklearn.tree import export_graphviz
graphviz.Source(export_graphviz(tree.dot))
!pip install pydot
!pip install pydot-ng
# Convert to png using system command (requires Graphviz)
from subprocess import call
call(['dot', '-Tpng', 'tree.dot', '-o', 'tree.png', '-Gdpi=600'])

# Display in jupyter notebook
from IPython.display import Image
Image(filename = 'tree.png')
#Convierto el dot a png
#! dot -Tpng tree.dot -o tree.png

from subprocess import call
call(['dot', '-Tpng', 'tree.dot', '-o', 'tree.png', '-Gdpi=600'])

#Ploteamos el png
img = cv2.imread('tree.png')
plt.figure(figsize = (20, 10))
plt.imshow(img)
#from subprocess import call
#call(['dot', '-Tpng', 'tree.dot', '-o', 'tree.png', '-Gdpi=600'])
#call(['dot', '-Tpng', 'tree.dot', '-o', 'tree.png'])
#Convierto el dot a png
dot -Tpng tree.dot -o tree.png

#Ploteamos el png
img = cv2.imread('tree.png')
plt.figure(figsize = (20, 10))
plt.imshow(img)


prediction_test = clf.predict(test_Df_x)
prediction_test
from sklearn.metrics import f1_score
f1_score(test_Df_y, prediction_test, average='micro')
accuracy_score(test_Df_y, prediction_test)
# Ajuste de Hiperparámetros para Algoritmo de Random forest

# Numero de arboles en random forest
n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]

# Numero de features a considerar en cada split
max_features = ['auto', 'sqrt']

# Numero Maximo de niveles en el arbol
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)

# Numero Minimo de samples requeridos para splitear un nodo
min_samples_split = [2, 5, 10]

# Numero Minimo de samples requerido por cada hoja nodo
min_samples_leaf = [1, 2, 4]

# Método de selección de samples para entrenamiento en cada arbol
bootstrap = [True, False]

# Crea la cuadrícula (random grid)
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}
pprint(random_grid)



# Uso del random grid para buscar los mejores hiperparámetros
# Primero crea el modelo de base para ajustar
rf = RandomForestRegressor()

# Busqueda aleatoria de parametros, usando 3 fold cross validation, 
# search across 100 diferentes combinaciones, y usa todos los nucleos disponibles (available score)
rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=True, random_state=42, n_jobs = -1)

# Fitea el modelo random search model
rf_random.fit(train_x, train_y)
print(accuracy_score(test_Df_y, rf_random))
# KNN

from sklearn.neighbors import KNeighborsClassifier

neigh = KNeighborsClassifier(n_neighbors=3)
neigh.fit(train_x, train_y)
knn=neigh.predict(test_Df_x)
print(accuracy_score(test_Df_y, knn))
# SVC with kernel 

from sklearn import svm
from sklearn.svm import SVC

rbf_svc = svm.SVC(kernel='rbf')
rbf_svc.fit(train_x, train_y)

rbf=rbf_svc.predict(test_Df_x)
accuracy_score(test_Df_y, rbf)

# AdaBoost

from sklearn.ensemble import AdaBoostClassifier

ad = AdaBoostClassifier(n_estimators=100, random_state=0)
ad.fit(train_x, train_y)
adb=ad.predict(test_Df_x)
print(accuracy_score(test_Df_y, adb))
# logistic

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(multi_class='ovr')
lr.fit(train_x, train_y)
lrc=lr.predict(test_Df_x)
print(accuracy_score(test_Df_y, lrc))
# Naive base Classifier

from sklearn.naive_bayes import GaussianNB

lr = GaussianNB()
lr.fit(train_x, train_y)
lrc=lr.predict(test_Df_x)
print(accuracy_score(test_Df_y, lrc))
# Decision Tree Classifier

from sklearn.tree import DecisionTreeClassifier 

dtree_model = DecisionTreeClassifier().fit(train_x, train_y)
dtree_predictions = dtree_model.predict(test_Df_x)
accuracy_score(test_Df_y, dtree_predictions)
# GradientBoostingClassifier

from sklearn.ensemble import GradientBoostingClassifier

gb=GradientBoostingClassifier()
gb.fit(train_x, train_y)
gbc=lr.predict(test_Df_x)
print(accuracy_score(test_Df_y, gbc))<jupyter_output><empty_output><jupyter_text>ALGORITMO 2 - CLASIFICACION Y PREDICCIÓN DE CLIENTES & DNN (RED NEURAL PROFUNDO)<jupyter_code># Limpiar datos y establecer tipos
df = limpieza_datos(df)
df = tipos_dataset(df)
df = agregar_fecha_log2(df)

#Plots a timeseries of total sales
df.groupby('date')['Quantity'].sum().plot()
#Prints the total number of days between start and end
print(df['date'].max() - df['date'].min())
#Dataset info
print(f'Total Number of Purchases: {df.shape[0]}')
print(f'Total Number of transactions: {df.Invoice.nunique()}')
print(f'Total Unique Days: {df.date.nunique()}')
print(f"Total Unique Customers: {df['Customer ID'].nunique()}")
print(f"We are predicting {(df['date'].max() - datetime(2011, 9, 11)).days} days")<jupyter_output>Total Number of Purchases: 301401
Total Number of transactions: 17451
Total Unique Days: 297
Total Unique Customers: 4278
We are predicting 80 days<jupyter_text>Preparado de Datos<jupyter_code>#Context data for the revenue (date & customerID)
id_lookup = df[['Customer ID', 'Invoice', 'date']].drop_duplicates()
id_lookup.index = id_lookup['Invoice']
id_lookup = id_lookup.drop('Invoice', axis=1)

transactions_data = pd.DataFrame(df.groupby('Invoice')['Revenue'].sum()).join(id_lookup)
#Split into train - test
rfm_train_test = calibration_and_holdout_data(transactions_data, 'Customer ID', 'date',
                                        calibration_period_end='2011-09-10',
                                        monetary_value_col = 'Revenue')   

#Selecting only customers with positive value in the calibration period (otherwise Gamma-Gamma model doesn't work)
rfm_train_test = rfm_train_test.loc[rfm_train_test['monetary_value_cal'] > 0, :]<jupyter_output><empty_output><jupyter_text>ML APPROACH1. Ingeniería de feature para entrenar y testear períodos2. Modelado3. Evaluación<jupyter_code>X_train, y_train = get_features(df, '2011-01-01', '2011-06-11', '2011-06-12', '2011-09-09')
X_test, y_test = get_features(df, '2011-04-02', '2011-09-10', '2011-09-11', '2011-12-09')
X_train, y_train = get_features(df, '2010-01-01', '2010-12-30', '2010-12-31', '2011-11-30')
X_train_, X_valid, y_train_, y_valid = train_test_split(X_train, y_train, random_state=42)

y_train_
scaler = StandardScaler()
X_train_ = scaler.fit_transform(X_train_)
X_valid = scaler.transform(X_valid)

X_test, y_test = get_features(df, '2010-01-01', '2010-12-30', '2010-12-31', '2011-11-30')
X_test = scaler.transform(X_test)
X_test
X_train_<jupyter_output><empty_output><jupyter_text>Modelado<jupyter_code>y_train
X_train
# The patience parameter is the amount of epochs to check for improvement
#early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=50)
early_stop = keras.callbacks.EarlyStopping(monitor='val_mse', patience=50)
model = build_model()
model.summary()
keras.utils.plot_model(model, "wide_and_deep_model.png", show_shapes=True)
hidden1 = model.layers[2]
print(hidden1.name)
weights, biases = hidden1.get_weights()
print(weights.shape)
weights[:,0].shape
# probemos epochs a partir de 100 en adelante
early_history = model.fit( X_train_, y_train_, 
                    epochs=1000, validation_data=(X_valid, y_valid), verbose=1,
                    callbacks=[early_stop, tfmodel.EpochDots()])<jupyter_output>Epoch 1/1000
46/46 [==============================] - 0s 4ms/step - loss: 58844044.0000 - mae: 2161.0393 - val_loss: 23248696.0000 - val_mae: 1958.7972
WARNING:tensorflow:Early stopping conditioned on metric `val_mse` which is not available. Available metrics are: loss,mae,val_loss,val_mae

Epoch: 0, loss:58844044.0000,  mae:2161.0393,  val_loss:23248696.0000,  val_mae:1958.7972,  
.Epoch 2/1000
46/46 [==============================] - 0s 3ms/step - loss: 59919284.0000 - mae: 2203.4790 - val_loss: 21942132.0000 - val_mae: 1921.5673
WARNING:tensorflow:Early stopping conditioned on metric `val_mse` which is not available. Available metrics are: loss,mae,val_loss,val_mae
.Epoch 3/1000
46/46 [==============================] - 0s 3ms/step - loss: 57358908.0000 - mae: 2149.9536 - val_loss: 23245788.0000 - val_mae: 1968.6476
WARNING:tensorflow:Early stopping conditioned on metric `val_mse` which is not available. Available metrics are: loss,mae,val_loss,val_mae
.Epoch 4/1000
46/46 [==========[...]<jupyter_text>EvaluaciónVeamos qué tan bien el modelo puede predecir los 3 próximos 3 meses que antes no había visto. Usaremos datos del período más reciente (X_test) para asegurarnos de que nuestro pronóstico sea lo más preciso posible.<jupyter_code>#Predicción
dnn_preds = model.predict(X_test).ravel()<jupyter_output><empty_output><jupyter_text>Explicación de datos<jupyter_code>#Dataset info
print(f'Total Número de Ventas: {df.shape[0]}')
print(f'Total Numero de transacciones: {df.InvoiceDate.nunique()}')
print(f'Total Días Unicos: {df.date.nunique()}')
print(f"Total clientes únicos: {df['Customer ID'].nunique()}")
print(f"Nuestra predicción {(df['date'].max() - datetime(2011, 9, 11)).days} days")
#Predicting
dnn_preds = model.predict(X_test).ravel()

evaluate(y_test, dnn_preds)<jupyter_output>Total Ventas Actual: 5142746.0
Total Ventas Predictivo: 3891386368.0
Individual R2 score: -351790.6943101874 
Individual Mean Absolute Error: 2005791.3451546184<jupyter_text>ALGORITMO 3 - PROPHET - PREDICCION SERIES DE TIEMPO & LSTM (RED NEURAL PROFUNDO PARA SERIES DE TIEMPO) Prophet is a procedure for forecasting time series data based on an additive model where non-linear trends are fit with yearly, weekly, and daily seasonality, plus holiday effects.  cargar solo un archivo para pruebasdf = carga_archivo()<jupyter_code># Limpiar datos y establecer tipos
df = limpieza_datos(df)
# Cuántos clientes compraron algo cada mes durante el último año
cantidad_mensual_pais(df)
# Rango de Precios comunes
rango_precios(df)
# Rango de cambio de precio durante el tiempo, enviamos los resultados de EDA 
customer_avg_spending, avg_selling_of_products = AED(df)

customer_avg_spending_insights(df)<jupyter_output><empty_output><jupyter_text>NLP<jupyter_code>class CleanColumn(BaseEstimator , TransformerMixin):
    def __init__(self):
        pass
    def fit(self ,X , y=None):
        return self
    def transform(self , X):
            description = [re.sub("\'" , "" , sent) for sent in X]
            description = [re.sub("\"" , "" , sent) for sent in description]
            description = [re.sub("\&" , "" , sent) for sent in description]
            description = [re.sub("\+" , "" , sent) for sent in description]
            description = [re.sub("\/" , "" , sent) for sent in description]
            description = [re.sub("\-" , "" , sent) for sent in description]
            description_clean =[re.sub("\d+" , "" , sent) for sent in description]
            return description_clean
class Debug(BaseEstimator , TransformerMixin):
    def fit(self , X , y=None , **fit_params):
        return self
    def transform(self , X):
        print(X.shape)
        return X  
nlp_pipeline = Pipeline([
    ('cleaning', CleanColumn()),
    ('vect', CountVectorizer()),
    ('debug', Debug()),
])
nlp_transformed = nlp_pipeline.fit_transform(df['Description'].astype(str))<jupyter_output><empty_output><jupyter_text>LDA - Crear target "y" desde feature "Description"<jupyter_code>LDA = LatentDirichletAllocation(n_components=5, random_state=42,verbose=True)
LDA.fit(nlp_transformed)
topic_values = LDA.transform(nlp_transformed)<jupyter_output><empty_output><jupyter_text>MODELADO  PROPHET<jupyter_code>class PrepareDataFrame(BaseException , TransformerMixin):
    def __init__(self , df):
        self.df = df
    def fit(self , df):
        return self
    def transform(self , df):
        df = df[['InvoiceDate', 'Topic', 'Price']]
        df['InvoiceDate'] = pd.to_datetime(df['InvoiceDate']).dt.strftime('%Y-%m-%d')
        
        df = df.groupby(['InvoiceDate', 'Topic']).agg({'Price':'mean'})
        df = df_modeling.reset_index()
        df_topic_1 = df[df['Topic'] == 1]
        df_topic_2 = df[df['Topic'] == 2]
        df_topic_3 = df[df['Topic'] == 3]
        df_topic_4 = df[df['Topic'] == 4]
        return df_topic_1 , df_topic_2,df_topic_3,df_topic_4
num_pipeline = Pipeline([
    ('preparedf' , PrepareDataFrame(df))
])
df_topic_1 , df_topic_2,df_topic_3,df_topic_4= num_pipeline.fit_transform(df)
list_topics = [df_topic_1 , df_topic_2, df_topic_3, df_topic_4]
for i in list_topics:
    create_prediction(i)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12999022256102613, 'fraction_numerical': 0.019820028835159008, 'mean_word_length': 4.3734639358860194, 'pattern_counts': {'":': 0, '<': 212, '<?xml version=': 0, '>': 216, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '52', 'dir': 'jupyter-structured-clean-dedup', 'id': '402710', 'license': 'MIT', 'n_tokens_mistral': 22351, 'n_tokens_neox': 21163, 'n_words': 4690, 'path': 'TP-EcoData-Borrador.ipynb', 'repo_name': 'hcgalvan/UNSAM-Machine-Learning-on-Economics', 'size': '60345'}