<jupyter_start><jupyter_text>archive to folder<jupyter_code>import urllib3
urllib3.disable_warnings()
# load function from Github
xfn = mlrun.import_function('https://raw.githubusercontent.com/yjb-ds/functions/arc2parq/fileutils/open_archive/function.yaml')

# configute it: mount on iguazio fabric, set as interactive (return stdout)
xfn.apply(mlrun.mount_v3io())
xfn.interactive = True

# create and run the task

images_path = '/User/mlrun/functions/images'

open_archive_task = mlrun.NewTask(
    'download',
    handler='open_archive', 
    params={'target_path': images_path,
            'key'        : 'contents'},
    inputs={'archive_url': 'http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip'}
)

# run
run = xfn.run(open_archive_task)<jupyter_output>[mlrun] 2020-01-20 08:36:14,989 starting run download uid=79a5b0f103c24367961cf8c107126dd2  -> http://mlrun-api:8080
[mlrun] 2020-01-20 08:36:15,069 Job is running in the background, pod: download-6mg4q
[mlrun] 2020-01-20 08:36:19,610 downloading http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip to local tmp
[mlrun] 2020-01-20 08:36:21,218 Verified directories
[mlrun] 2020-01-20 08:36:21,218 Extracting zip
[mlrun] 2020-01-20 08:36:22,988 extracted archive to content
[mlrun] 2020-01-20 08:36:23,001 log artifact content at content, size: None, db: Y

[mlrun] 2020-01-20 08:36:23,011 run executed, status=completed
final state: succeeded<jupyter_text>_________  archive to parquet  load and configure function<jupyter_code># load function from Github
xfn = mlrun.import_function('https://raw.githubusercontent.com/yjb-ds/functions/arc2parq/fileutils/arc_to_parquet/arc_to_parquet.yaml')

# configure function: mount on the Iguazio data fabric, set as interactive (return stdout)
xfn.apply(mlrun.mount_v3io())
xfn.interactive = True<jupyter_output><empty_output><jupyter_text>deploy / build The following triggers a build when run for the first time using specs found in the yaml file above.  Unless that file changes, this only needs to be run once, even after the notebook has been restarted:<jupyter_code>xfn.deploy()<jupyter_output>[mlrun] 2020-01-20 05:35:07,015 starting remote build, image: .mlrun/func-default-arc_to_parquet-latest
[36mINFO[0m[0000] Resolved base name python:3.6-jessie to python:3.6-jessie 
[36mINFO[0m[0000] Resolved base name python:3.6-jessie to python:3.6-jessie 
[36mINFO[0m[0000] Downloading base image python:3.6-jessie     
[36mINFO[0m[0000] Error while retrieving image from cache: getting file info: stat /cache/sha256:0318d80cb241983eda20b905d77fa0bfb06e29e5aabf075c7941ea687f1c125a: no such file or directory 
[36mINFO[0m[0000] Downloading base image python:3.6-jessie     
[36mINFO[0m[0000] Built cross stage deps: map[]                
[36mINFO[0m[0000] Downloading base image python:3.6-jessie     
[36mINFO[0m[0000] Error while retrieving image from cache: getting file info: stat /cache/sha256:0318d80cb241983eda20b905d77fa0bfb06e29e5aabf075c7941ea687f1c125a: no such file or directory 
[36mINFO[0m[0000] Downloading base image python:3.6-jessie     
[36mINFO[0m[0001] Unpa[...]<jupyter_text>Also note that the build time can be reduced if you specifiy a pre-built image with all required packages.<jupyter_code># useful constants
target_path = '/User/mlrun/functions/parquet'
archive = 'https://fpsignals-public.s3.amazonaws.com/x_test_50.csv.gz'
parquet_file = 'x_test_50.parquet' # the file extension is not necessary
parquet_file_path = target_path + "/" + parquet_file
artifact_key = 'raw_data'
# create and run the task
arc_to_parq_task = mlrun.NewTask(
    'arc2parq', 
    handler='arc_to_parquet',  # a string since we are calling this 'remotely', outside this notebook
    params={
        'target_path': target_path,
        'name'       : parquet_file, 
        'key'        : artifact_key,
        'archive_url': archive},
    outputs=[artifact_key])

# run
run = xfn.run(arc_to_parq_task)<jupyter_output>[mlrun] 2020-01-20 05:38:21,743 starting run arc2parq uid=42af41d93f294cd09aace4942d25b106  -> http://mlrun-api:8080
[mlrun] 2020-01-20 05:38:21,823 Job is running in the background, pod: arc2parq-96gmq
[mlrun] 2020-01-20 05:38:37,072 destination file exists
[mlrun] 2020-01-20 05:38:37,072 logging /User/mlrun/functions/parquet/x_test_50.parquet to context
[mlrun] 2020-01-20 05:38:37,083 log artifact raw_data at /User/mlrun/functions/parquet/x_test_50.parquet, size: None, db: Y

[mlrun] 2020-01-20 05:38:37,094 run executed, status=completed
final state: succeeded<jupyter_text>___  tests<jupyter_code>import os
import numpy as np
import pandas as pd
# add more context tests
# convert these to real tests
assert artifact_key in run.outputs.keys(), f"mlrun.functions: key {artifact_key} not fond in outputs"
assert os.path.isfile(parquet_file_path),  f"mlrun.functions: artifact source not found at {parquet_file_path}"
original = pd.read_csv(archive)
copied   = pd.read_parquet(parquet_file_path, engine="pyarrow")
assert np.array_equal(original, copied),   "mlrun.functions: original and copied data not equal"<jupyter_output><empty_output><jupyter_text>cleanup<jupyter_code>os.remove(parquet_file_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11121801039060997, 'fraction_numerical': 0.09620935154897056, 'mean_word_length': 5.4812967581047385, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '321530', 'license': 'Apache-2.0', 'n_tokens_mistral': 2201, 'n_tokens_neox': 1856, 'n_words': 435, 'path': 'tests/file-utils.ipynb', 'repo_name': 'Sharon-iguazio/functions', 'size': '5197'}