<jupyter_start><jupyter_text>MercadoLibre Challenge  TensorflowHub embeddings<jupyter_code>#!pip install tensorflow-gpu==2.0.0b0 --user
#!pip install sklearn --user
### OJO: Reiniciar el kernel despues de instalar esto
#!wget https://meli-data-challenge.s3.amazonaws.com/train.csv.gz
#!wget https://meli-data-challenge.s3.amazonaws.com/test.csv
#!gunzip train.csv.gz
#!mkdir saved_models
EXPERIMENT_NAME = "meli-TFH" 
EXPERIMENT_VERSION = "v3_1"
LOG_DIR = "logs/TFH-3"<jupyter_output><empty_output><jupyter_text>Version log TFH-v3_1Es la versión 3, final del challenge, ejecutada en GPU de muestra con el objetivo de tener una versión en Jupyter que se pueda compartir y analizar mas fácilmente.  TFH-v2_3Viene de 2_2_2 TFH-v2_2Ajustar hiperparámetros para buscar mejor score TFH-v2_1Usar normalización TFH-v2_0Nueva arquitectura, incorporarar info de lenguajeViene de meli-TFH-v1_1  Development  Initialize<jupyter_code>%matplotlib inline

import sys
import os

#os.environ['CUDA_VISIBLE_DEVICES'] = '-1' #disable gpu

import time
import pickle
import numpy as np
import pandas as pd
from sklearn.metrics import balanced_accuracy_score 

import tensorflow as tf
import tensorflow_hub as hub
import matplotlib.pyplot as plt

from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Embedding, Dense, Dropout
from tensorflow.keras.layers import Concatenate, Input
from tensorflow.keras.optimizers import RMSprop, Adam
from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint
from tensorflow.keras.utils import to_categorical
print(f"TF Version: {tf.__version__}")
print(f"TF Hub Version: {hub.__version__}")
assert( tf.__version__ >= "2.0.0-beta0" ) #rc1
assert( hub.__version__ >= "0.3" ) #"0.7.0.dev"
#initialize env
#seeds to make reproducible
#todo: check reproducibility
np.random.seed(12347)
tf.random.set_seed(12347)

pd.options.display.max_rows = 10

#filenames and directories
DATASET_FN = "train.csv"
ROWS_TO_LOAD = None#4000000 #None == all
SAVED_MODEL_DIR = "saved_models/"

#TFHUB_EMB_MODEL = "https://tfhub.dev/google/universal-sentence-encoder/2" 
# download the module manually if network problems 
# like URLError: <urlopen error [Errno -3] Temporary failure in name resolution>
# check https://www.tensorflow.org/hub/common_issues 
#TFHUB_EMB_MODEL = "../tf_hub_cache/gnews-swivel-20dim-v1"
#TFHUB_EMB_MODEL = "../tf_hub_cache/nnlm-es-dim50" #
#TFHUB_EMB_MODEL = "https://tfhub.dev/google/tf2-preview/nnlm-en-dim128/1"
#TFHUB_EMB_MODEL = "https://tfhub.dev/google/tf2-preview/nnlm-es-dim128/1"
#TFHUB_EMB_MODEL = "../tf_hub_cache/nnlm-es-dim128-with-normalization"
TFHUB_EMB_MODEL = "https://tfhub.dev/google/tf2-preview/nnlm-es-dim128-with-normalization/1"
TFHUB_EMB_MODEL_DIM = 128
LANGUAGE = "spanish" #"portuguese" #None

MODEL_DENSE_UNITS = 2048
MODEL_DROPOUT_RATE = 0.5

#Training
TRAIN_EPOCHS = 3
TRAIN_INITIAL_EPOCH = 0
TRAIN_BATCH_SIZE = 8192

#set some parameters on how data will be used
# specify a small proportion to speed things while testing, 1.0 when running full training
DATASET_PROPORTION_TO_USE = 1.0
# how much data will reserve for test set (of the DS prop to use) (0.10)
TEST_SET_SPLIT = 0.004
# how much of the data will be used for validation (of the DS prop to use) (0.05)
VALIDATION_SET_SPLIT = 0.004<jupyter_output><empty_output><jupyter_text>Load and prepare datasets<jupyter_code>df = pd.read_csv(DATASET_FN, nrows=ROWS_TO_LOAD)
if LANGUAGE is not None:
    df = df[df["language"]==LANGUAGE]
print(df)
output_dim = len(df["category"].unique())
print(output_dim)
print(list(df["category"][:3]))
cat_dict = dict(zip(df["category"].unique(), np.arange(output_dim)))
inverse_cat_dict = dict(zip(cat_dict.values(), cat_dict.keys()))
labels = df["category"].map(cat_dict)
#labels = to_categorical(df["category"].map(cat_dict))
labels[:3]
language = (df["language"]==LANGUAGE).astype("int32")<jupyter_output><empty_output><jupyter_text>Split datasets<jupyter_code>num_samples = len(df)
num_test_samples = int(num_samples * TEST_SET_SPLIT)
num_training_samples = num_samples - num_test_samples

training_set_data = df["title"].head(num_training_samples)
training_set_lang = language[:num_training_samples]
training_set_labels = labels[:num_training_samples]
test_set_data = df["title"].tail(num_test_samples)
test_set_lang = language[-num_test_samples:]
test_set_labels = labels[-num_test_samples:]

print(training_set_data.shape)
print(training_set_labels.shape)
print(test_set_data.shape)
print(test_set_labels.shape)<jupyter_output>(9960000,)
(9960000,)
(40000,)
(40000,)<jupyter_text>Build Model<jupyter_code>hub_layer = hub.KerasLayer(TFHUB_EMB_MODEL,
                    input_shape=[],
                    output_shape=[TFHUB_EMB_MODEL_DIM],
                    trainable=True, 
                    dtype=tf.string)

input_lang = Input(shape=(1,))

mod = Sequential()
mod.add(hub_layer)

fo = Dense(MODEL_DENSE_UNITS, activation="relu", name="DEN_1")(mod.output)
fo = Dropout(rate=MODEL_DROPOUT_RATE)(fo)
fo = Dense(MODEL_DENSE_UNITS, activation="relu", name="DEN_2")(fo)
fo = Dropout(rate=MODEL_DROPOUT_RATE)(fo)
fo = Concatenate()([fo, input_lang])
fo = Dense(output_dim, activation="softmax", name="DEN_OUT")(fo)
model = Model(inputs=[mod.input, input_lang], outputs=fo)

optimizer = Adam()
model.compile(optimizer=optimizer, 
              loss='sparse_categorical_crossentropy')

print(model.summary())<jupyter_output>Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
keras_layer_input (InputLayer)  [(None,)]            0                                            
__________________________________________________________________________________________________
keras_layer (KerasLayer)        (None, 128)          125009920   keras_layer_input[0][0]          
__________________________________________________________________________________________________
DEN_1 (Dense)                   (None, 2048)         264192      keras_layer[0][0]                
__________________________________________________________________________________________________
dropout (Dropout)               (None, 2048)         0           DEN_1[0][0]                  [...]<jupyter_text>Train and Test<jupyter_code>#run params
runid = "%s-%s-%s"%(EXPERIMENT_NAME,
     EXPERIMENT_VERSION,
     time.strftime(time.strftime('%y%m%d_%H%M',time.localtime())))

#Create saved model dir     
directory = SAVED_MODEL_DIR+"/"+runid
if not os.path.exists(directory):
    os.makedirs(directory)
    
print("Python: "+str(sys.version))
print("Tensorflow version: "+tf.__version__)
print("Keras version: "+tf.keras.__version__)
print("RUNID: "+runid)
#Train from saved point
#model = tf.keras.models.load_model("../saved_models/meli-TFH-v3-191001_0150/model.hdf5", 
#                                            custom_objects={"KerasLayer":hub_layer})

print('Training...')
tensorboard = TensorBoard(log_dir=LOG_DIR+'/'+runid)
checkpoint = ModelCheckpoint(directory+"/model.hdf5", monitor='val_loss',
                             verbose=1, save_best_only=True, mode="min")

# train
t0 = time.time()
print("Start:"+time.strftime("%Y%m%d_%H%M",time.localtime()))
history = model.fit([training_set_data.array, training_set_lang.array], 
          training_set_labels,
          batch_size=TRAIN_BATCH_SIZE,
          epochs=TRAIN_EPOCHS,
          initial_epoch=TRAIN_INITIAL_EPOCH,
          validation_split=VALIDATION_SET_SPLIT,
          verbose=1,
          callbacks=[tensorboard, checkpoint])

tfin = time.time()
print("End:" + time.strftime("%Y%m%d_%H%M",time.localtime()))
print(tfin-t0)
fn = directory+"/history.pickle"
with open(fn, "wb")as fo:
    pickle.dump(history.history, fo, protocol=4)
print(f"Saved {fn}")
#Last Model (current)
#analysis_model = model

#Best Model saved
analysis_model = tf.keras.models.load_model(directory+"/model.hdf5", 
                                            custom_objects={"KerasLayer":hub_layer})

print("Predict:")
predictions = analysis_model.predict([test_set_data.array, 
                                      test_set_lang.array], verbose=1)

inverse_cat_dict = dict(zip(cat_dict.values(), cat_dict.keys()))
predicted_categories = [inverse_cat_dict[np.argmax(p)] for p in predictions]

real_categories = [inverse_cat_dict[p] for p in test_set_labels]

bac = balanced_accuracy_score(real_categories, predicted_categories)
print(f"Eval BAC={bac}")<jupyter_output>Predict:
40000/40000 [==============================] - 17s 428us/sample
Eval BAC=0.825535295632827<jupyter_text>History graphs<jupyter_code>#with open(directory+"/history.pickle", "rb") as f:
#    history = pickle.load(f)
history_dict = history.history #current
loss = history_dict['loss']
val_loss = history_dict['val_loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'r', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

fn = directory+"/history.pdf"
plt.show()
#Save
#plt.savefig(fn)
#print(f"Saved {fn}")<jupyter_output><empty_output><jupyter_text>Create Submission<jupyter_code>submission_test_df = pd.read_csv("test.csv")
#submission_test_df
sub_data = submission_test_df["title"]
sub_lang = (submission_test_df["language"]=='spanish').astype("int32")

sub_predictions = analysis_model.predict([sub_data.array, sub_lang.array], verbose=1)
sub_predicted_categories = [inverse_cat_dict[np.argmax(p)] for p in sub_predictions]
#sub_predicted_categories[:10]
submission = pd.DataFrame({"id":range(len(sub_predicted_categories)), 
                          "category":sub_predicted_categories})
#submission
fn = directory+"/submission-"+runid+".csv"
submission[["id","category"]].to_csv(fn, index=False)
print(f"Saved {fn}")<jupyter_output>246955/246955 [==============================] - 102s 414us/sample
Saved saved_models//meli-TFH-v3_1-191008_1642/submission-meli-TFH-v3_1-191008_1642.csv
{'fraction_non_alphanumeric': 0.13838145747397368, 'fraction_numerical': 0.027499508937340406, 'mean_word_length': 5.119591346153846, 'pattern_counts': {'":': 4, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 7, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '157457', 'license': 'MIT', 'n_tokens_mistral': 3695, 'n_tokens_neox': 3486, 'n_words': 618, 'path': 'meli_TFH_v3_1.ipynb', 'repo_name': 'eduardofv/meli2019', 'size': '10182'}