<jupyter_start><jupyter_text>Face Recognition Using mlrun with OpenCV And PyTorch A complete pipeline of data processing, model training and serving function deployment.  Declare global variables<jupyter_code>ARTIFACTS_PATH = '/User/demos/demos/faces/artifacts/'
CODE_PATH = '/User/demos/demos/faces/code/'
use_gpu = False<jupyter_output><empty_output><jupyter_text>Import and define mlrun functions for the pipeline<jupyter_code># nuclio: ignore
from mlrun import new_function, code_to_function, NewTask, mount_v3io
import kfp
from kfp import dsl
encode_fn = code_to_function('encode-images', image='yjbds/mlrun-horovod-gpu:0.4.5', kind='job', filename=CODE_PATH + 'encode.py')
encode_fn.export(CODE_PATH + 'encode.yaml')
encode_fn.apply(mount_v3io())
if use_gpu:
    train_fn = new_function(name='train', image='yjbds/mlrun-horovod-gpu:0.4.5', kind='mpijob', command=CODE_PATH + 'horovod_train.py')
    train_fn.gpus(1)

else:
    train_fn = new_function(name='train', image='yjbds/mlrun-horovod-gpu:0.4.5', kind='job', command=CODE_PATH + 'train.py') 

train_fn.export(CODE_PATH + 'train.yaml')
train_fn.spec.replicas = 2
train_fn.apply(mount_v3io())
model_serving_function = code_to_function(
    name='recognize-faces', 
    filename='./nuclio-face-prediction.ipynb',
    kind='nuclio')

model_serving_function.apply(mount_v3io())
api_serving_function = code_to_function(
    name='video-api-server', 
    filename='./nuclio-api-serving.ipynb',
    kind='nuclio')

api_serving_function.apply(mount_v3io())
from mlrun import mlconf
mlconf.dbpath = 'http://mlrun-api:8080'<jupyter_output><empty_output><jupyter_text>Create pipeline<jupyter_code>@dsl.pipeline(
    name='face recognition pipeline',
    description='Creates and deploys a face recognition model'
)
def face_recognition_pipeline(with_cuda=False, with_horovod=False):
    
    encode = encode_fn.as_step(name='encode-images', handler='encode_images', artifact_path=ARTIFACTS_PATH, outputs=['idx2name', 'encodings_path'],
                       inputs={'cuda': with_cuda})
    
    train = train_fn.as_step(name='train', artifact_path=ARTIFACTS_PATH, outputs=['model'], 
                               inputs={'processed_data': encode.outputs['encodings_path']})
    
    deploy_model = model_serving_function.deploy_step(project='default', models={'face_rec_v1': train.outputs['model']})
    
    deploy_api = api_serving_function.deploy_step(project='default').after(deploy_model)
    
client = kfp.Client(namespace='default-tenant')
#For debug purposes compile pipeline code
kfp.compiler.Compiler().compile(face_recognition_pipeline, 'face_rec.yaml')<jupyter_output><empty_output><jupyter_text>Run pipeline<jupyter_code>arguments = {}
run_result = client.create_run_from_pipeline_func(face_recognition_pipeline, arguments=arguments, run_name='face_rec_1', experiment_name='face_rec')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12842762929538354, 'fraction_numerical': 0.00798333911836168, 'mean_word_length': 6.277777777777778, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '418349', 'license': 'Apache-2.0', 'n_tokens_mistral': 1011, 'n_tokens_neox': 990, 'n_words': 148, 'path': 'faces/code/face-recognition.ipynb', 'repo_name': 'royeis/demos', 'size': '2881'}