# Path: openaddr/jobs.py
# def setup_logger(logfile = None, log_level = logging.DEBUG, log_stderr = True, log_config_file = "~/.openaddr-logging.json"):
#     ''' Set up logging for openaddr code.
#         If the file ~/.openaddr-logging.json exists, it will be used as a DictConfig
#         Otherwise a default configuration will be set according to function parameters.
#         Default is to log DEBUG and above to stderr, and nothing to a file.
#     '''
#     # Get a handle for the openaddr logger and its children
#     openaddr_logger = logging.getLogger('openaddr')
# 
#     # Default logging format. {0} will be replaced with a destination-appropriate timestamp
#     log_format = '%(process)06s  {0} %(levelname)06s: %(message)s'
# 
#     # Set the logger level to show everything, and filter down in the handlers.
#     openaddr_logger.setLevel(logging.DEBUG)
# 
#     # Remove all previously installed handlers
#     for old_handler in openaddr_logger.handlers:
#         openaddr_logger.removeHandler(old_handler)
# 
#     # Tell multiprocessing to log messages as well. Multiprocessing can interact strangely
#     # with logging, see http://bugs.python.org/issue23278
#     mp_logger = multiprocessing.get_logger()
#     mp_logger.propagate=True
# 
#     log_config_file = os.path.expanduser(log_config_file)
#     if os.path.exists(log_config_file):
#         # Use a JSON config file in the user's home directory if it exists
#         # See http://victorlin.me/posts/2012/08/26/good-logging-practice-in-python
#         log_config_dict = json.load(file(log_config_file))
#         # Override this flag; needs to be set for our module-level loggers to work.
#         log_config_dict['disable_existing_loggers'] = False
#         logging.config.dictConfig(log_config_dict)
#         openaddr_logger.info("Using logger config at %s", log_config_file)
#     else:
#         # No config file? Set up some sensible defaults
# 
#         # Set multiprocessing level as requested
#         mp_logger.setLevel(log_level)
# 
#         everything_logger = logging.getLogger()
#         # Set up a logger to stderr
#         if log_stderr:
#             handler1 = logging.StreamHandler()
#             handler1.setLevel(log_level)
#             handler1.setFormatter(logging.Formatter(log_format.format('%(relativeCreated)10.1f')))
#             everything_logger.addHandler(handler1)
#         # Set up a logger to a file
#         if logfile:
#             handler2 = logging.FileHandler(logfile, mode='w')
#             handler2.setLevel(log_level)
#             handler2.setFormatter(logging.Formatter(log_format.format('%(asctime)s')))
#             everything_logger.addHandler(handler2)
# 
# Path: openaddr/parcels/utils.py
# def fetch(url, filepath):
#     r = requests.get(url, stream=True)
#     with open(filepath, 'wb') as f:
#         for chunk in r.iter_content(chunk_size=1024):
#             if chunk:  # filter out keep-alive new chunks
#                 f.write(chunk)
#                 f.flush()
# 
#     return filepath
# 
# def unzip(filepath, dest):
#     with zipfile.ZipFile(filepath) as zf:
#         zf.extractall(dest)
# 
# def rlistdir(path):
#     """
#     Recursively return all files in path.
# 
#     Does not follow symlinks.
#     """
#     files = []
#     for dirpath, dirnames, filenames in os.walk(path):
#         for f in filenames:
#             files.append(os.path.join(dirpath, f))
# 
#     return files
# 
# def import_with_fiona(fpath, source):
#     """
#     Use fiona to import a parcel file.
# 
#     Return a list of dict objects containing WKT-formatted geometries in
#     addition to any metadata.
#     """
#     shapes = []
# 
#     try:
#         with fiona.Env():
#             data = fiona.open(fpath)
#             for obj in data:
#                 try:
#                     shape = scrape_fiona_metadata(obj, source)
#                     geom = to_shapely_obj(obj)
#                     if geom:
#                         shape['geom'] = dumps(geom)
#                         shapes.append(shape)
#                 except Exception as e:
#                     _L.warning('error loading shape from fiona. {}'.format(e))
#     except Exception as e:
#         _L.warning('error importing file. {}'.format(e))
# 
#     return shapes
# 
# def import_csv(fpath, source):
#     """
#     Import a csv file
# 
#     Return a list of dict objects containing WKT-formatted geometries in
#     addition to any metadata.
#     """
# 
#     data = []
#     try:
#         csvdata = []
#         with open(fpath, 'r') as f:
#             statereader = csv.reader(f, delimiter=',')
#             for row in statereader:
#                 csvdata.append(row)
#         header = csvdata.pop(0)
#         for row in csvdata:
#             try:
#                 shape = scrape_csv_metadata(row, header, source)
#                 shape['geom'] = row[header.index('OA:geom')]
#                 data.append(shape)
#             except Exception as e:
#                 _L.warning('error loading shape from csv. {}'.format(e))
#     except Exception as e:
#         _L.warning('error importing csv. {}'.format(e))
# 
#     return data


import csv
import logging
import os
import re
import shutil
import sys
import traceback
from . import config
from openaddr.jobs import setup_logger
from shapely.wkt import dumps
from .utils import fetch, unzip, rlistdir, import_with_fiona, import_csv



_L = logging.getLogger('openaddr.parcels')

csv.field_size_limit(sys.maxsize)

def parse_source(source, idx, header):
    """
    Import data from a single source based on the data type.
    """
    path = '{}/{}'.format(config.workspace_dir, idx)
    if not os.path.exists(path):
        os.makedirs(path)

    cache_url = source[header.index('cache')]
    cache_filename = re.search('/[^/]*$', cache_url).group()
    fetch(cache_url, path + cache_filename)


# Path: openaddr/jobs.py
# def setup_logger(logfile = None, log_level = logging.DEBUG, log_stderr = True, log_config_file = "~/.openaddr-logging.json"):
#     ''' Set up logging for openaddr code.
#         If the file ~/.openaddr-logging.json exists, it will be used as a DictConfig
#         Otherwise a default configuration will be set according to function parameters.
#         Default is to log DEBUG and above to stderr, and nothing to a file.
#     '''
#     # Get a handle for the openaddr logger and its children
#     openaddr_logger = logging.getLogger('openaddr')
# 
#     # Default logging format. {0} will be replaced with a destination-appropriate timestamp
#     log_format = '%(process)06s  {0} %(levelname)06s: %(message)s'
# 
#     # Set the logger level to show everything, and filter down in the handlers.
#     openaddr_logger.setLevel(logging.DEBUG)
# 
#     # Remove all previously installed handlers
#     for old_handler in openaddr_logger.handlers:
#         openaddr_logger.removeHandler(old_handler)
# 
#     # Tell multiprocessing to log messages as well. Multiprocessing can interact strangely
#     # with logging, see http://bugs.python.org/issue23278
#     mp_logger = multiprocessing.get_logger()
#     mp_logger.propagate=True
# 
#     log_config_file = os.path.expanduser(log_config_file)
#     if os.path.exists(log_config_file):
#         # Use a JSON config file in the user's home directory if it exists
#         # See http://victorlin.me/posts/2012/08/26/good-logging-practice-in-python
#         log_config_dict = json.load(file(log_config_file))
#         # Override this flag; needs to be set for our module-level loggers to work.
#         log_config_dict['disable_existing_loggers'] = False
#         logging.config.dictConfig(log_config_dict)
#         openaddr_logger.info("Using logger config at %s", log_config_file)
#     else:
#         # No config file? Set up some sensible defaults
# 
#         # Set multiprocessing level as requested
#         mp_logger.setLevel(log_level)
# 
#         everything_logger = logging.getLogger()
#         # Set up a logger to stderr
#         if log_stderr:
#             handler1 = logging.StreamHandler()
#             handler1.setLevel(log_level)
#             handler1.setFormatter(logging.Formatter(log_format.format('%(relativeCreated)10.1f')))
#             everything_logger.addHandler(handler1)
#         # Set up a logger to a file
#         if logfile:
#             handler2 = logging.FileHandler(logfile, mode='w')
#             handler2.setLevel(log_level)
#             handler2.setFormatter(logging.Formatter(log_format.format('%(asctime)s')))
#             everything_logger.addHandler(handler2)
# 
# Path: openaddr/parcels/utils.py
# def fetch(url, filepath):
#     r = requests.get(url, stream=True)
#     with open(filepath, 'wb') as f:
#         for chunk in r.iter_content(chunk_size=1024):
#             if chunk:  # filter out keep-alive new chunks
#                 f.write(chunk)
#                 f.flush()
# 
#     return filepath
# 
# def unzip(filepath, dest):
#     with zipfile.ZipFile(filepath) as zf:
#         zf.extractall(dest)
# 
# def rlistdir(path):
#     """
#     Recursively return all files in path.
# 
#     Does not follow symlinks.
#     """
#     files = []
#     for dirpath, dirnames, filenames in os.walk(path):
#         for f in filenames:
#             files.append(os.path.join(dirpath, f))
# 
#     return files
# 
# def import_with_fiona(fpath, source):
#     """
#     Use fiona to import a parcel file.
# 
#     Return a list of dict objects containing WKT-formatted geometries in
#     addition to any metadata.
#     """
#     shapes = []
# 
#     try:
#         with fiona.Env():
#             data = fiona.open(fpath)
#             for obj in data:
#                 try:
#                     shape = scrape_fiona_metadata(obj, source)
#                     geom = to_shapely_obj(obj)
#                     if geom:
#                         shape['geom'] = dumps(geom)
#                         shapes.append(shape)
#                 except Exception as e:
#                     _L.warning('error loading shape from fiona. {}'.format(e))
#     except Exception as e:
#         _L.warning('error importing file. {}'.format(e))
# 
#     return shapes
# 
# def import_csv(fpath, source):
#     """
#     Import a csv file
# 
#     Return a list of dict objects containing WKT-formatted geometries in
#     addition to any metadata.
#     """
# 
#     data = []
#     try:
#         csvdata = []
#         with open(fpath, 'r') as f:
#             statereader = csv.reader(f, delimiter=',')
#             for row in statereader:
#                 csvdata.append(row)
#         header = csvdata.pop(0)
#         for row in csvdata:
#             try:
#                 shape = scrape_csv_metadata(row, header, source)
#                 shape['geom'] = row[header.index('OA:geom')]
#                 data.append(shape)
#             except Exception as e:
#                 _L.warning('error loading shape from csv. {}'.format(e))
#     except Exception as e:
#         _L.warning('error importing csv. {}'.format(e))
# 
#     return data


# Path: openaddr/parcels/parse.py
import csv
import logging
import os
import re
import shutil
import sys
import traceback
from . import config
from openaddr.jobs import setup_logger
from shapely.wkt import dumps
from .utils import fetch, unzip, rlistdir, import_with_fiona, import_csv




_L = logging.getLogger('openaddr.parcels')

csv.field_size_limit(sys.maxsize)

def parse_source(source, idx, header):
    """
    Import data from a single source based on the data type.
    """
    path = '{}/{}'.format(config.workspace_dir, idx)
    if not os.path.exists(path):
        os.makedirs(path)

    cache_url = source[header.index('cache')]
    cache_filename = re.search('/[^/]*$', cache_url).group()
    fetch(cache_url, path + cache_filename)


    files = rlistdir(path)