<jupyter_start><jupyter_text>End-to-end NVIDIA Merlin Recommender Sysem with Vertex AI.This notebook shows how deploy and execute an end-to-end Vertex Pipeline to run the NVIDIA Merlin recommendation system.The notebook covers the following:1. Training pipeline overview.2. Set pipeline configurations.3. Build pipeline container images.4. Configure pipeline parameters.5. Compile KFP pipeline.6. Submit pipeline to Vertex AI.  1. Training Pipeline Overview  Setup<jupyter_code>import os
import json
from datetime import datetime
from google.cloud import aiplatform as vertex_ai
from kfp.v2 import compiler
PROJECT_ID = 'merlin-on-gcp' # Change to your project Id.
REGION = 'us-central1' # Change to your region.
BUCKET =  'merlin-on-gcp' # Change to your bucket.

MODEL_NAME = 'deepfm'
MODEL_VERSION = 'v01'
MODEL_DISPLAY_NAME = f'criteo-hugectr-{MODEL_NAME}-{MODEL_VERSION}'
WORKSPACE = f'gs://{BUCKET}/{MODEL_DISPLAY_NAME}'
TRAINING_PIPELINE_NAME = f'merlin-training-pipeline'

BQ_DATASET_NAME = 'criteo_pipeline' # Set to your BigQuery dataset including the Criteo dataset.
BQ_LOCATION = 'us' # Set to your BigQuery dataset location.
BQ_TRAIN_TABLE_NAME = 'train'
BQ_VALID_TABLE_NAME = 'valid'

NVT_IMAGE_NAME = 'nvt_preprocessing'
NVT_IMAGE_URI = f'gcr.io/{PROJECT_ID}/{NVT_IMAGE_NAME}'
NVT_DOCKERFILE = 'src/Dockerfile.nvtabular'

HUGECTR_IMAGE_NAME = 'hugectr_training'
HUGECTR_ITMAGE_URI = f'gcr.io/{PROJECT_ID}/{HUGECTR_IMAGE_NAME}'
HUGECTR_DOCKERFILE = 'src/Dockerfile.hugectr'<jupyter_output><empty_output><jupyter_text>2. Set Pipeline Configurations<jupyter_code>os.environ['PROJECT_ID'] = PROJECT_ID
os.environ['REGION'] = REGION
os.environ['BUCKET'] = BUCKET
os.environ['WORKSPACE'] = WORKSPACE

os.environ['BQ_DATASET_NAME'] = BQ_DATASET_NAME
os.environ['BQ_LOCATION'] = BQ_LOCATION
os.environ['BQ_TRAIN_TABLE_NAME'] = BQ_TRAIN_TABLE_NAME
os.environ['BQ_VALID_TABLE_NAME'] = BQ_VALID_TABLE_NAME

os.environ['TRAINING_PIPELINE_NAME'] = TRAINING_PIPELINE_NAME
os.environ['MODEL_NAME'] = MODEL_NAME
os.environ['MODEL_VERSION'] = MODEL_VERSION
os.environ['MODEL_DISPLAY_NAME'] = MODEL_DISPLAY_NAME

os.environ['NVT_IMAGE_URI'] = NVT_IMAGE_URI
os.environ['HUGECTR_ITMAGE_URI'] = HUGECTR_ITMAGE_URI

os.environ['MEMORY_LIMIT'] = '120G'
os.environ['CPU_LIMIT'] = '32'
os.environ['GPU_LIMIT'] = '4'
os.environ['GPU_TYPE'] = 'nvidia-tesla-t4'

os.environ['MACHINE_TYPE'] = 'a2-highgpu-4g'
os.environ['ACCELERATOR_TYPE'] = 'NVIDIA_TESLA_A100'
os.environ['ACCELERATOR_NUM'] = '4'
os.environ['NUM_WORKERS'] = '12'

os.environ['NUM_SLOTS'] = '26'
os.environ['MAX_NNZ'] = '2'
os.environ['EMBEDDING_VECTOR_SIZE'] = '11'
os.environ['MAX_BATCH_SIZE'] = '64'
os.environ['MODEL_REPOSITORY_PATH'] = '/models'<jupyter_output><empty_output><jupyter_text>3. Build Pipeline Container Images<jupyter_code>! gcloud builds submit --timeout "2h" --tag {IMAGE_URI} {NVT_DOCKERFILE} --machine-type=e2-highcpu-8
! gcloud builds submit --timeout "2h" --tag {IMAGE_URI} {HUGECTR_DOCKERFILE} --machine-type=e2-highcpu-8<jupyter_output><empty_output><jupyter_text>4. Configure pipeline parameters<jupyter_code>NUM_EPOCHS = 0
MAX_ITERATIONS = 50000
EVAL_INTERVAL = 1000
EVAL_BATCHES = 500
EVAL_BATCHES_FINAL = 2500
DISPLAY_INTERVAL = 200
SNAPSHOT_INTERVAL = 0
PER_GPU_BATCHSIZE = 2048
LR = 0.001
DROPOUT_RATE = 0.5
parameter_values = {
    'shuffle': json.dumps(None) # select PER_PARTITION, PER_WORKER, FULL, or None.
    'per_gpu_batch_size': PER_GPU_BATCHSIZE,
    'max_iter': MAX_ITERATIONS,
    'max_eval_batches': EVAL_BATCHES ,
    'eval_batches': EVAL_BATCHES_FINAL ,
    'dropout_rate': DROPOUT_RATE,
    'lr': LR ,
    'num_epochs': NUM_EPOCHS,
    'eval_interval': EVAL_INTERVAL,
    'snapshot': SNAPSHOT_INTERVAL,
    'display_interval': DISPLAY_INTERVAL
}<jupyter_output><empty_output><jupyter_text>5. Compile KFP pipeline<jupyter_code>from src.pipelines.training_pipelines import training_bq

compiled_pipeline_path = 'merlin_training_bq.json'
compiler.Compiler().compile(
       pipeline_func=training_bq,
       package_path=compiled_pipeline_path
)<jupyter_output><empty_output><jupyter_text>6. Submit pipeline to Vertex AI<jupyter_code>job_name = f'merlin_training_bq_{datetime.now().strftime("%Y%m%d%H%M%S")}'

pipeline_job = vertex_ai.PipelineJob(
    display_name=job_name,
    template_path=compiled_pipeline_path,
    enable_caching=False,
    parameter_values=parameter_values,
)

pipeline_job.run()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1177797833935018, 'fraction_numerical': 0.016922382671480145, 'mean_word_length': 6.790861159929701, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '465889', 'license': 'Apache-2.0', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1763, 'n_words': 291, 'path': '04-e2e-pipeline.ipynb', 'repo_name': 'jarokaz/merlin-on-vertex', 'size': '4432'}