<commit_before>import itertools

from datapackage_pipelines.wrapper import ingest, spew
from datapackage_pipelines.utilities.resource_matcher import ResourceMatcher

parameters, datapackage, resource_iterator = ingest()

sources = ResourceMatcher(parameters.get('sources'))

target = parameters.get('target', {})
if 'name' not in target:
    target['name'] = 'concat'
if 'path' not in target:
    target['path'] = 'data/' + target['name'] + '.csv'
target.update(dict(
    mediatype='text/csv',
    schema=dict(fields=[], primaryKey=[])
))

fields = parameters['fields']

# Create mapping between source field names to target field names
field_mapping = {}
for target_field, source_fields in fields.items():
    if source_fields is not None:
        for source_field in source_fields:
            if source_field in field_mapping:
                raise RuntimeError('Duplicate appearance of %s (%r)' % (source_field, field_mapping))
            field_mapping[source_field] = target_field

    if target_field in field_mapping:
        raise RuntimeError('Duplicate appearance of %s' % target_field)

    field_mapping[target_field] = target_field

# Create the schema for the target resource
needed_fields = sorted(fields.keys())
for resource in datapackage['resources']:
    if not sources.match(resource['name']):
        continue

    schema = resource.get('schema', {})
    pk = schema.get('primaryKey', [])
    for field in schema.get('fields', []):
        orig_name = field['name']
        if orig_name in field_mapping:
            name = field_mapping[orig_name]
            if name not in needed_fields:
                continue
            if orig_name in pk:
                target['schema']['primaryKey'].append(name)
            target['schema']['fields'].append(field)
            field['name'] = name
            needed_fields.remove(name)

for name in needed_fields:
    target['schema']['fields'].append(dict(
        name=name, type='string'
    ))

# Update resources in datapackage (make sure they are consecutive)
prefix = True
suffix = False
num_concatenated = 0
new_resources = []
for resource in datapackage['resources']:
    name = resource['name']
    match = sources.match(name)
    if prefix:
        if match:
            prefix = False
            num_concatenated += 1
        else:
            new_resources.append(resource)
    elif suffix:<commit_msg>Handle empty primaryKey in concatenate<commit_after>import itertools

from datapackage_pipelines.wrapper import ingest, spew
from datapackage_pipelines.utilities.resource_matcher import ResourceMatcher

parameters, datapackage, resource_iterator = ingest()

sources = ResourceMatcher(parameters.get('sources'))

target = parameters.get('target', {})
if 'name' not in target:
    target['name'] = 'concat'
if 'path' not in target:
    target['path'] = 'data/' + target['name'] + '.csv'
target.update(dict(
    mediatype='text/csv',
    schema=dict(fields=[], primaryKey=[])
))

fields = parameters['fields']

# Create mapping between source field names to target field names
field_mapping = {}
for target_field, source_fields in fields.items():
    if source_fields is not None:
        for source_field in source_fields:
            if source_field in field_mapping:
                raise RuntimeError('Duplicate appearance of %s (%r)' % (source_field, field_mapping))
            field_mapping[source_field] = target_field

    if target_field in field_mapping:
        raise RuntimeError('Duplicate appearance of %s' % target_field)

    field_mapping[target_field] = target_field

# Create the schema for the target resource
needed_fields = sorted(fields.keys())
for resource in datapackage['resources']:
    if not sources.match(resource['name']):
        continue

    schema = resource.get('schema', {})
    pk = schema.get('primaryKey', [])
    for field in schema.get('fields', []):
        orig_name = field['name']
        if orig_name in field_mapping:
            name = field_mapping[orig_name]
            if name not in needed_fields:
                continue
            if orig_name in pk:
                target['schema']['primaryKey'].append(name)
            target['schema']['fields'].append(field)
            field['name'] = name
            needed_fields.remove(name)

if len(target['schema']['primaryKey']) == 0:
    del target['schema']['primaryKey']

for name in needed_fields:
    target['schema']['fields'].append(dict(
        name=name, type='string'
    ))

# Update resources in datapackage (make sure they are consecutive)
prefix = True
suffix = False
num_concatenated = 0
new_resources = []
for resource in datapackage['resources']:
    name = resource['name']
    match = sources.match(name)
    if prefix:
        if match:
            prefix = False
            num_concatenated += 1
        else:
            new_resources.append(resource)
    elif suffix:
{'fraction_non_alphanumeric': 0.11127049180327869, 'fraction_numerical': 0.0010245901639344263, 'mean_word_length': 3.2591623036649215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '659606', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1331, 'n_words': 374}