<jupyter_start><jupyter_text>pipeline parameters<jupyter_code># kv = ws.get_default_keyvault()
# print(len(kv.get_secret(pipeline_kv_readapi)))
pipeline_inpart = PipelineParameter(name="pipeline_inpart", default_value='2020/08/10')
pipeline_metadata = PipelineParameter(name="pipeline_metadata", default_value='20200810_images.csv')
pipeline_kv_customimg = PipelineParameter(name="pipeline_kv_customimg", default_value='api-custom-vision')
pipeline_kv_readapi = PipelineParameter(name="pipeline_kv_readapi", default_value='api-readapi')
pipeline_dataset_param = PipelineParameter(name='pipeline_dataset_id', default_value=input_images_dataset)
pipeline_dataset_output = PipelineParameter(name='pipeline_output_dataset_id', default_value=output_data_path)


# reference (pipeline_dataset_output, DataPathComputeBinding())
print(pipeline_inpart)
print(pipeline_dataset_param)<jupyter_output>PipelineParameter_Name:pipeline_inpart_Default:2020/08/10
PipelineParameter_Name:pipeline_dataset_id_Default:FileDataset
{
  "source": [
    "('godzilla', 'images')"
  ],
  "definition": [
    "GetDatastoreFiles"
  ]
}<jupyter_text>python envinronment configuration<jupyter_code># conda dependencies 
env_name = 'MAG-ParallelRunEnv'
print(env_name in ws.environments)
if env_name in ws.environments: 
    env = ws.environments.get(env_name)
    if env and type(env) is Environment: 
        print('Found environment: ' + env_name)
else: 
    print('environment not found, refer to 01_config_notebook to register envinroment...')<jupyter_output>True
Found environment: MAG-ParallelRunEnv<jupyter_text>parallel run config and step<jupyter_code>parallelrunconfig = ParallelRunConfig(
    environment=env, 
    entry_script='minibatch_process.py', 
    error_threshold=1,
    output_action='append_row',
    compute_target=compute_target, 
    node_count=1, 
    process_count_per_node=2,
    mini_batch_size='2',
    source_directory='scripts', 
    description='description of batch step config',
    logging_level='INFO'
)



#parallelrunconfig('script', 'other stuff')
parallelrunstep = ParallelRunStep(
    name='cv-detection-batch-dataset-step', 
    parallel_run_config=parallelrunconfig, 
    inputs=[DatasetConsumptionConfig('dataset_param_config', pipeline_dataset_param).as_mount()], 
    # inputs=[ Dataset.File.from_files((godzilla_datastore, 'images')).as_named_input('anpr_images').as_mount()],
    # inputs=[]
    side_inputs=[metadata_config], 
    output=step_output_dir,
    arguments=[
        '--input_partition', pipeline_inpart, 
        '--metadata_config', metadata_config,
        '--metadata', pipeline_metadata,
        '--kv_customimage', pipeline_kv_customimg, 
        '--kv_readapi', pipeline_kv_readapi], 
    allow_reuse=False
)


# ('config', 'inputs as mount', 'arguments passing in pipeline args')<jupyter_output><empty_output><jupyter_text>data transfer step to move score data from out temp storage to blog storagehttps://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-data-transfer.ipynbhttps://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-showcasing-datapath-and-pipelineparameter.ipynb<jupyter_code>adf_target


transfer_output_toblob = DataTransferStep(
    name="transfer output to blob", 
    source_data_reference=step_output_dir, 
    source_reference_type='directory',
    destination_data_reference=(pipeline_dataset_output, DataPathComputeBinding()), 
    destination_reference_type='directory', 
    compute_target=adf_target
)

transfer_output_toblob.run_after(parallelrunstep)
transfer_output_toblob<jupyter_output><empty_output><jupyter_text>prepare pipeline 1. run pipeline2. create a draft pipeline: draft pipelines represent a mutable pipeline which can be used to submit runs and create Published Pipelines. draft pipelines can be iterated using `PipelineDrafts` 2. publish and print end point<jupyter_code>pipeline = Pipeline(workspace=ws, steps=[parallelrunstep,transfer_output_toblob])



# pipelinedraft = PipelineDraft.create(
#     workspace=ws, 
#     pipeline=pipeline,
#     name= f'ANPR Batch Scoring {strDate} {strTime}', 
#     experiment_name='ANPR Batch Scoring', 
#     continue_on_step_failure=True, 
#     tags={'dev': 'true'}, 
#     properties={'date': strDate, 'time': strTime}
# )
# pipeline_draft = PipelineDraft.create(workspace=ws,
#                                          name="TestPipelineDraft",
#                                          description="draft description",
#                                          experiment_name="helloworld",
#                                          pipeline=pipeline,
#                                          continue_on_step_failure=True,
#                                          tags={'dev': 'true'},
#                                          properties={'train': 'value'})
pipeline_run = Experiment(ws, 'MAG-batch-paramdataset').submit(pipeline)
pipeline_run.wait_for_completion(show_output=True)
dtnow = datetime.now()
strDate = dtnow.strftime('%Y/%m/%d')
strTime = dtnow.strftime('%H:%M')
published_pipeline = pipeline_run.publish_pipeline(
    name='MAG-batchscore-dataset', 
    description=f'published pipeline {strDate} {strTime} ', 
    version='2.0', 
    continue_on_step_failure=True)
print(published_pipeline.endpoint)<jupyter_output>https://australiaeast.api.azureml.ms/pipelines/v1.0/subscriptions/907c8efc-c2c8-4c49-a4e1-aeb880e10c88/resourceGroups/aml/providers/Microsoft.MachineLearningServices/workspaces/magaml/PipelineRuns/PipelineSubmit/01bb90bc-0b7a-4d38-b7ec-aae9808df60d
{'fraction_non_alphanumeric': 0.10851213506858952, 'fraction_numerical': 0.012486809708054871, 'mean_word_length': 4.761904761904762, 'pattern_counts': {'":': 2, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '607180', 'license': 'MIT', 'n_tokens_mistral': 1791, 'n_tokens_neox': 1744, 'n_words': 247, 'path': 'batch/03_create_pipeline_datasetparam.ipynb', 'repo_name': 'mutazag/cv', 'size': '5686'}