# Path: loom/util.py
# def mkdir_p(dirname):
#     'like mkdir -p'
#     if not os.path.exists(dirname):
#         try:
#             os.makedirs(dirname)
#         except OSError as e:
#             if not os.path.exists(dirname):
#                 raise e
# 
# def rm_rf(path):
#     'like rm -rf'
#     if os.path.exists(path):
#         if os.path.isdir(path):
#             shutil.rmtree(path)
#         else:
#             os.remove(path)
# 
# def parallel_map(fun, args):
#     if not isinstance(args, list):
#         args = list(args)
#     is_daemon = multiprocessing.current_process().daemon
#     if THREADS == 1 or len(args) < 2 or is_daemon:
#         print 'Running {} in this thread'.format(fun.__name__)
#         return map(fun, args)
#     else:
#         print 'Running {} in {:d} threads'.format(fun.__name__, THREADS)
#         pool = multiprocessing.Pool(THREADS)
#         fun_args = [(fun, arg) for arg in args]
#         return pool.map(print_trace, fun_args, chunksize=1)


import os
import re
import parsable
import loom.datasets
import loom.tasks
    import boto
    import boto
from itertools import izip
from loom.util import mkdir_p, rm_rf, parallel_map
        conn = boto.connect_s3().get_bucket(bucket)
        key = conn.get_key(source)
        key.get_contents_to_filename(destin)
        print 'finished {}'.format(source)
    except:
        rm_rf(destin)
        raise


@parsable.command
def download(s3_url=S3_URL):
    '''
    Download dataset from S3 and load into loom.benchmark jig.
    '''
    bucket, path = s3_split(s3_url)
    conn = boto.connect_s3().get_bucket(bucket)
    keys = [
        key.name
        for key in conn.list(path)
        if re.match(r'.*\d\d\d\.csv\.gz$', key.name)
    ]
    assert keys, 'nothing to download'
    files = [os.path.join(ROWS_CSV, os.path.basename(key)) for key in keys]
    tasks = [
        (bucket, source, destin)
        for source, destin in izip(keys, files)
        if not os.path.exists(destin)
    ]
    if tasks:
        print 'starting download of {} files'.format(len(tasks))

# Path: loom/util.py
# def mkdir_p(dirname):
#     'like mkdir -p'
#     if not os.path.exists(dirname):
#         try:
#             os.makedirs(dirname)
#         except OSError as e:
#             if not os.path.exists(dirname):
#                 raise e
# 
# def rm_rf(path):
#     'like rm -rf'
#     if os.path.exists(path):
#         if os.path.isdir(path):
#             shutil.rmtree(path)
#         else:
#             os.remove(path)
# 
# def parallel_map(fun, args):
#     if not isinstance(args, list):
#         args = list(args)
#     is_daemon = multiprocessing.current_process().daemon
#     if THREADS == 1 or len(args) < 2 or is_daemon:
#         print 'Running {} in this thread'.format(fun.__name__)
#         return map(fun, args)
#     else:
#         print 'Running {} in {:d} threads'.format(fun.__name__, THREADS)
#         pool = multiprocessing.Pool(THREADS)
#         fun_args = [(fun, arg) for arg in args]
#         return pool.map(print_trace, fun_args, chunksize=1)


# Path: examples/taxi/main.py
import os
import re
import parsable
import loom.datasets
import loom.tasks
    import boto
    import boto
from itertools import izip
from loom.util import mkdir_p, rm_rf, parallel_map

        conn = boto.connect_s3().get_bucket(bucket)
        key = conn.get_key(source)
        key.get_contents_to_filename(destin)
        print 'finished {}'.format(source)
    except:
        rm_rf(destin)
        raise


@parsable.command
def download(s3_url=S3_URL):
    '''
    Download dataset from S3 and load into loom.benchmark jig.
    '''
    bucket, path = s3_split(s3_url)
    conn = boto.connect_s3().get_bucket(bucket)
    keys = [
        key.name
        for key in conn.list(path)
        if re.match(r'.*\d\d\d\.csv\.gz$', key.name)
    ]
    assert keys, 'nothing to download'
    files = [os.path.join(ROWS_CSV, os.path.basename(key)) for key in keys]
    tasks = [
        (bucket, source, destin)
        for source, destin in izip(keys, files)
        if not os.path.exists(destin)
    ]
    if tasks:
        print 'starting download of {} files'.format(len(tasks))

        mkdir_p(ROWS_CSV)