# Path: datapackage_pipelines/utilities/extended_json.py
# class json(object):
#     dumps = _dumps
#     loads = _loads
#     dumpl = _dumpl
#     loadl = _loadl
#     dump = _dump
#     load = _load
#     JSONDecodeError = _json.JSONDecodeError
# 
# Path: datapackage_pipelines/wrapper/input_processor.py
# def process_input(infile, validate=False, debug=False):
#     dependency_dp = read_json(infile, True)
#     dp = read_json(infile)
#     resources = dp.get('resources', [])
#     original_resources = copy.deepcopy(resources)
# 
#     if len(dp.get('resources', [])) == 0:
#         # Currently datapackages with no resources are disallowed in the schema.
#         # Since this might happen in the early stages of a pipeline,
#         # we're adding this hack to avoid validation errors
#         dp_to_validate = copy.deepcopy(dp)
#         dp_to_validate['resources'] = [{
#             'name': '__placeholder__',
#             'path': PATH_PLACEHOLDER
#         }]
#     else:
#         dp_to_validate = dp
#     try:
#         datapackage.validate(dp_to_validate)
#     except ValidationError as e:
#         logging.info('FAILED TO VALIDATE %r', dp_to_validate)
#         for e in e.errors:
#             try:
#                 logging.error("Data Package validation error: %s at dp%s",
#                               e.message,
#                               "[%s]" % "][".join(repr(index) for index in e.path))
#             except AttributeError:
#                 logging.error("Data Package validation error: %s", e)
#         raise
# 
#     infile.readline().strip()
# 
#     def resources_iterator(_resources, _original_resources):
#         # we pass a resource instance that may be changed by the processing
#         # code, so we must keep a copy of the original resource (used to
#         # validate incoming data)
#         ret = []
#         for resource, orig_resource in zip(_resources, _original_resources):
#             if not streaming(resource):
#                 continue
# 
#             res_iter = ResourceIterator(infile,
#                                         resource, orig_resource,
#                                         validate, debug)
#             ret.append(res_iter)
#         return iter(ret)
# 
#     return dp, resources_iterator(resources, original_resources), dependency_dp
# 
# Path: datapackage_pipelines/utilities/resources.py
# def streaming(descriptor):
#     return descriptor.get(PROP_STREAMING)


import gzip
import sys
import os
import logging
from contextlib import ExitStack, redirect_stderr, redirect_stdout
from tableschema.exceptions import CastError
from ..utilities.extended_json import json
from .input_processor import process_input
from ..utilities.resources import streaming
# ## LOW LEVEL INTERFACE

def _ingest(debug=False):
    global cache
    global first
    params = None
    validate = False
    if len(sys.argv) > 4:
        first = sys.argv[1] == '0'
        params = json.loads(sys.argv[2])
        validate = sys.argv[3] == 'True'
        cache = sys.argv[4]

    datapackage, resource_iterator, dependency_dp = process_input(sys.stdin, validate, debug)
    dependency_datapackage_urls.update(dependency_dp)

    return params, datapackage, resource_iterator


def spew(dp, resources_iterator, stats=None, finalizer=None):
    files = [stdout]

    cache_filename = ''
    if len(cache) > 0:
        if not os.path.exists('.cache'):
            os.mkdir('.cache')
        cache_filename = os.path.join('.cache', cache)
        files.append(gzip.open(cache_filename+'.ongoing', 'wt'))

    expected_resources = \

# Path: datapackage_pipelines/utilities/extended_json.py
# class json(object):
#     dumps = _dumps
#     loads = _loads
#     dumpl = _dumpl
#     loadl = _loadl
#     dump = _dump
#     load = _load
#     JSONDecodeError = _json.JSONDecodeError
# 
# Path: datapackage_pipelines/wrapper/input_processor.py
# def process_input(infile, validate=False, debug=False):
#     dependency_dp = read_json(infile, True)
#     dp = read_json(infile)
#     resources = dp.get('resources', [])
#     original_resources = copy.deepcopy(resources)
# 
#     if len(dp.get('resources', [])) == 0:
#         # Currently datapackages with no resources are disallowed in the schema.
#         # Since this might happen in the early stages of a pipeline,
#         # we're adding this hack to avoid validation errors
#         dp_to_validate = copy.deepcopy(dp)
#         dp_to_validate['resources'] = [{
#             'name': '__placeholder__',
#             'path': PATH_PLACEHOLDER
#         }]
#     else:
#         dp_to_validate = dp
#     try:
#         datapackage.validate(dp_to_validate)
#     except ValidationError as e:
#         logging.info('FAILED TO VALIDATE %r', dp_to_validate)
#         for e in e.errors:
#             try:
#                 logging.error("Data Package validation error: %s at dp%s",
#                               e.message,
#                               "[%s]" % "][".join(repr(index) for index in e.path))
#             except AttributeError:
#                 logging.error("Data Package validation error: %s", e)
#         raise
# 
#     infile.readline().strip()
# 
#     def resources_iterator(_resources, _original_resources):
#         # we pass a resource instance that may be changed by the processing
#         # code, so we must keep a copy of the original resource (used to
#         # validate incoming data)
#         ret = []
#         for resource, orig_resource in zip(_resources, _original_resources):
#             if not streaming(resource):
#                 continue
# 
#             res_iter = ResourceIterator(infile,
#                                         resource, orig_resource,
#                                         validate, debug)
#             ret.append(res_iter)
#         return iter(ret)
# 
#     return dp, resources_iterator(resources, original_resources), dependency_dp
# 
# Path: datapackage_pipelines/utilities/resources.py
# def streaming(descriptor):
#     return descriptor.get(PROP_STREAMING)


# Path: datapackage_pipelines/wrapper/wrapper.py
import gzip
import sys
import os
import logging
from contextlib import ExitStack, redirect_stderr, redirect_stdout
from tableschema.exceptions import CastError
from ..utilities.extended_json import json
from .input_processor import process_input
from ..utilities.resources import streaming

# ## LOW LEVEL INTERFACE

def _ingest(debug=False):
    global cache
    global first
    params = None
    validate = False
    if len(sys.argv) > 4:
        first = sys.argv[1] == '0'
        params = json.loads(sys.argv[2])
        validate = sys.argv[3] == 'True'
        cache = sys.argv[4]

    datapackage, resource_iterator, dependency_dp = process_input(sys.stdin, validate, debug)
    dependency_datapackage_urls.update(dependency_dp)

    return params, datapackage, resource_iterator


def spew(dp, resources_iterator, stats=None, finalizer=None):
    files = [stdout]

    cache_filename = ''
    if len(cache) > 0:
        if not os.path.exists('.cache'):
            os.mkdir('.cache')
        cache_filename = os.path.join('.cache', cache)
        files.append(gzip.open(cache_filename+'.ongoing', 'wt'))

    expected_resources = \

        len(list(filter(streaming, dp.get('resources', []))))