# Path: loom/util.py
# def mkdir_p(dirname):
#     'like mkdir -p'
#     if not os.path.exists(dirname):
#         try:
#             os.makedirs(dirname)
#         except OSError as e:
#             if not os.path.exists(dirname):
#                 raise e
# 
# def rm_rf(path):
#     'like rm -rf'
#     if os.path.exists(path):
#         if os.path.isdir(path):
#             shutil.rmtree(path)
#         else:
#             os.remove(path)
# 
# def parallel_map(fun, args):
#     if not isinstance(args, list):
#         args = list(args)
#     is_daemon = multiprocessing.current_process().daemon
#     if THREADS == 1 or len(args) < 2 or is_daemon:
#         print 'Running {} in this thread'.format(fun.__name__)
#         return map(fun, args)
#     else:
#         print 'Running {} in {:d} threads'.format(fun.__name__, THREADS)
#         pool = multiprocessing.Pool(THREADS)
#         fun_args = [(fun, arg) for arg in args]
#         return pool.map(print_trace, fun_args, chunksize=1)


import os
import re
import parsable
import loom.datasets
import loom.tasks
    import boto
    import boto
from itertools import izip
from loom.util import mkdir_p, rm_rf, parallel_map
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


S3_URL = 's3://pk-dsp/taxi-data/partitioned/geocoded'

ROOT = os.path.dirname(os.path.abspath(__file__))
SCHEMA = os.path.join(ROOT, 'schema.json')
EXAMPLE = os.path.join(ROOT, 'example.csv')
ROWS_CSV = os.path.join(ROOT, 'rows_csv')


def s3_split(url):
    bucket, path = re.match(r's3://([^/]*)/(.*)', S3_URL).group(1, 2)
    return bucket, path


def s3_get((bucket, source, destin)):
    try:
        print 'starting {}'.format(source)
        conn = boto.connect_s3().get_bucket(bucket)
        key = conn.get_key(source)
        key.get_contents_to_filename(destin)
        print 'finished {}'.format(source)
    except:

# Path: loom/util.py
# def mkdir_p(dirname):
#     'like mkdir -p'
#     if not os.path.exists(dirname):
#         try:
#             os.makedirs(dirname)
#         except OSError as e:
#             if not os.path.exists(dirname):
#                 raise e
# 
# def rm_rf(path):
#     'like rm -rf'
#     if os.path.exists(path):
#         if os.path.isdir(path):
#             shutil.rmtree(path)
#         else:
#             os.remove(path)
# 
# def parallel_map(fun, args):
#     if not isinstance(args, list):
#         args = list(args)
#     is_daemon = multiprocessing.current_process().daemon
#     if THREADS == 1 or len(args) < 2 or is_daemon:
#         print 'Running {} in this thread'.format(fun.__name__)
#         return map(fun, args)
#     else:
#         print 'Running {} in {:d} threads'.format(fun.__name__, THREADS)
#         pool = multiprocessing.Pool(THREADS)
#         fun_args = [(fun, arg) for arg in args]
#         return pool.map(print_trace, fun_args, chunksize=1)


# Path: examples/taxi/main.py
import os
import re
import parsable
import loom.datasets
import loom.tasks
    import boto
    import boto
from itertools import izip
from loom.util import mkdir_p, rm_rf, parallel_map

# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


S3_URL = 's3://pk-dsp/taxi-data/partitioned/geocoded'

ROOT = os.path.dirname(os.path.abspath(__file__))
SCHEMA = os.path.join(ROOT, 'schema.json')
EXAMPLE = os.path.join(ROOT, 'example.csv')
ROWS_CSV = os.path.join(ROOT, 'rows_csv')


def s3_split(url):
    bucket, path = re.match(r's3://([^/]*)/(.*)', S3_URL).group(1, 2)
    return bucket, path


def s3_get((bucket, source, destin)):
    try:
        print 'starting {}'.format(source)
        conn = boto.connect_s3().get_bucket(bucket)
        key = conn.get_key(source)
        key.get_contents_to_filename(destin)
        print 'finished {}'.format(source)
    except:

        rm_rf(destin)