<jupyter_start><jupyter_text>Machine Learning PipelineIn this notebook, we show how to run the flowers classification workflow as a pipeline  Set up<jupyter_code>%pip install --upgrade -q kfp
# CHANGE AS needed
REGION = 'us-central1'  # Change as needed to a region where you have quota
# Find this value from https://console.cloud.google.com/ai-platform/pipelines/clusters
KFPHOST = '7cac804811a5e5d0-dot-us-central2.pipelines.googleusercontent.com'
PROJECT = !gcloud config get-value project
PROJECT = PROJECT[0]
print(PROJECT)
%env PROJECT = {PROJECT}
%env REGION = {REGION}
BUCKET = PROJECT + "-flowers-pipeline"
%env BUCKET = {BUCKET}
!gsutil mb -l {REGION} gs://{BUCKET}<jupyter_output><empty_output><jupyter_text>Build the container<jupyter_code>%%capture --no-stderr
!../build_docker_image.sh<jupyter_output><empty_output><jupyter_text>Convert JPEG files to TF Records<jupyter_code>%%writefile components/create_dataset.yaml
name: create_dataset
description: Converts JPEG files to TensorFlow Records using Dataflow or Apache Beam
inputs:
- {name: runner, type: str, default: 'DirectRunner', description: 'DirectRunner or DataflowRunner'}
- {name: project_id, type: str, description: 'Project to bill Dataflow job to'}
- {name: region, type: str, description: 'Region to run Dataflow job in'}
- {name: input_csv, type: GCSPath, description: 'Path to CSV file'}
- {name: output_dir, type: GCSPath, description: 'Top-level directory for TF records'}
- {name: labels_dict, type: GCSPath, description: 'Dictionary file for class names'}
outputs:
- {name: tfrecords_topdir, type: GCSPath, description: 'Top-level directory for TF records'}
implementation:
  container:
    image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest
    command: [
        "bash", "/src/practical-ml-vision-book/07_training/components/create_dataset.sh"
    ]
    args: [
        {inputValue: output_dir},
        {outputPath: tfrecords_topdir},
        "--all_data", {inputValue: input_csv},
        "--labels_file", {inputValue: labels_dict},
        "--project_id", {inputValue: project_id},
        "--output_dir", {inputValue: output_dir},
        "--runner", {inputValue: runner},
        "--region", {inputValue: region},
    ]
%%writefile components/noop_create_dataset.yaml
name: noop_create_dataset
description: Converts JPEG files to TensorFlow Records using Dataflow or Apache Beam
inputs:
- {name: runner, type: str, default: 'DirectRunner', description: 'DirectRunner or DataflowRunner'}
- {name: project_id, type: str, description: 'Project to bill Dataflow job to'}
- {name: region, type: str, description: 'Region to run Dataflow job in'}
- {name: input_csv, type: GCSPath, description: 'Path to CSV file'}
- {name: output_dir, type: GCSPath, description: 'Top-level directory for TF records'}
- {name: labels_dict, type: GCSPath, description: 'Dictionary file for class names'}
outputs:
- {name: tfrecords_topdir, type: GCSPath, description: 'Top-level directory for TF records'}
implementation:
  container:
    image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest
    command: [
        "bash", "/src/practical-ml-vision-book/07_training/components/noop_create_dataset.sh"
    ]
    args: [
        {inputValue: output_dir},
        {outputPath: tfrecords_topdir}
    ]<jupyter_output>Overwriting components/noop_create_dataset.yaml<jupyter_text>Train modelTo do it locally on the cluster instead of on CAIP, we'll use gcloud local traininggcloud ai-platform local train --package-path $PACKAGE_PATH        --module-name $MODULE_NAME --job-dir ${JOB_DIR}_local        -- --num_training_examples 100 --with_color_distort False --crop_ratio 0.6<jupyter_code>%%writefile components/train_model_kfp.yaml
name: train_model_kfp
description: Trains an ML model on KFP
inputs:
- {name: input_topdir, type: GCSPath, description: 'Top-level directory for TF records'}
- {name: region, type: str, description: 'Region (ignored)'}
- {name: job_dir, type: GCSPath, description: 'Top-level output directory'}
outputs:
- {name: trained_model, type: GCSPath, description: 'location of trained model'}
implementation:
  container:
    image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest
    command: [
        "bash", "/src/practical-ml-vision-book/07_training/components/train_model_kfp.sh", 
    ]
    args: [
        {inputValue: input_topdir},
        {inputValue: region},
        {inputValue: job_dir},
        {outputPath: trained_model},
    ]
%%writefile components/train_model_caip.yaml
name: train_model_caip
description: Trains an ML model on CAIP
inputs:
- {name: input_topdir, type: GCSPath, description: 'Top-level directory for TF records'}
- {name: region, type: str, description: 'Region'}
- {name: job_dir, type: GCSPath, description: 'Top-level output directory'}
outputs:
- {name: trained_model, type: GCSPath, description: 'location of trained model'}
implementation:
  container:
    image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest
    command: [
        "bash", "/src/practical-ml-vision-book/07_training/components/train_model_caip.sh", 
    ]
    args: [
        {inputValue: input_topdir},
        {inputValue: region},
        {inputValue: job_dir},
        {outputPath: trained_model},
    ]<jupyter_output>Overwriting components/train_model_caip.yaml<jupyter_text>The pipeline<jupyter_code>import kfp
import kfp.dsl as dsl
import json
import os

create_dataset_op = kfp.components.load_component_from_file(
    #'components/noop_create_dataset.yaml'
    'components/create_dataset.yaml'
)
train_model_op = kfp.components.load_component_from_file(
    #'components/train_model_kfp.yaml'
    'components/train_model_caip.yaml'
)
deploy_op = kfp.components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/master/components/gcp/ml_engine/deploy/component.yaml')

@dsl.pipeline(
    name='Flowers Transfer Learning Pipeline',
    description='End-to-end pipeline'
)
def flowerstxf_pipeline(
    project_id = PROJECT,
    bucket = BUCKET,
    region = REGION
):
    # Step 1: Create dataset
    create_dataset = create_dataset_op(
        runner='DataflowRunner',
        project_id=project_id,
        region=region,
        input_csv='gs://cloud-ml-data/img/flower_photos/all_data.csv',
        output_dir='gs://{}/data/flower_tfrecords'.format(bucket),
        labels_dict='gs://cloud-ml-data/img/flower_photos/dict.txt'
    )
    create_dataset.execution_options.caching_strategy.max_cache_staleness = "P7D"
    
    # Step 2: Train model
    train_model = train_model_op(
        input_topdir=create_dataset.outputs['tfrecords_topdir'],
        region=region,
        job_dir='gs://{}/trained_model'.format(bucket)
    )
    train_model.execution_options.caching_strategy.max_cache_staleness = "P0D"
    
    # Step 3: Deploy trained model
    deploy_model = deploy_op(
        model_uri=train_model.outputs['trained_model'],
        project_id=project_id, 
        model_id='flowers', 
        version_id='txf', 
        runtime_version='2.3', 
        python_version='3.7',
        version={}, 
        replace_existing_version='True', 
        set_default='True', 
        wait_interval='30')<jupyter_output><empty_output><jupyter_text>Compile and submit pipeline<jupyter_code>pipeline_func = flowerstxf_pipeline
pipeline_filename = pipeline_func.__name__ + '.zip'
import kfp.compiler as compiler
compiler.Compiler().compile(pipeline_func, pipeline_filename)
import kfp
client = kfp.Client(host=KFPHOST)
experiment = client.create_experiment('from_notebook')
run_name = pipeline_func.__name__ + ' run'
run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename, 
                                 {
                                     'project_id': PROJECT,
                                     'bucket': BUCKET,
                                     'region': REGION
                                 })<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11458333333333333, 'fraction_numerical': 0.004643574297188755, 'mean_word_length': 3.958929682638457, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '449980', 'license': 'Apache-2.0', 'n_tokens_mistral': 2561, 'n_tokens_neox': 2504, 'n_words': 568, 'path': '07_training/07e_mlpipeline.ipynb', 'repo_name': 'qwerlarlgus/practical-ml-vision-book', 'size': '7968'}