<commit_before># -*- coding: utf-8 -*-
#
#  patch.py
#  csvdiff
#

"""
The the patch format.
"""

import sys
import json

from . import records
from . import error


SCHEMA = {
    '$schema': 'http://json-schema.org/draft-04/schema#',
    'title': 'csvdiff',
    'description': 'The patch format used by csvdiff.',
    'type': 'object',
    'properties': {
        '_index': {
            'type': 'array',
            'minItems': 1,
            'items': {'type': 'string'},
        },
        'added': {
            'type': 'array',
            'items': {'type': 'object'},
        },
        'removed': {
            'type': 'array',
            'items': {'type': 'object'},
        },
        'changed': {
            'type': 'array',
            'items': {
                'type': 'object',
                'properties': {
                    'key': {'type': 'array',
                            'items': {'type': 'string'},
                            'minItems': 1},
                    'fields': {
                        'type': 'object',
                    },
                    'minProperties': 1,
                    'patternProperties': {
                        '.+': {'type': 'object',
                               'properties': {
                                   'from': {'type': 'string'},
                                   'to': {'type': 'string'},
                               },
                               'required': ['from', 'to']}
                    },
                },
            },
        },
    },
    'required': ['_index', 'added', 'changed', 'removed'],
}


def is_valid(diff):
    "Validate it against the schema."
    pass


def apply(diff, recs, strict=True):
    """
    Transform the records with the patch. May fail if the records do not
    match those expected in the patch.
    """
    index_columns = diff['_index']
    indexed = records.index(recs, index_columns, strict=strict)
    _add_records(indexed, diff['added'], index_columns, strict=strict)
    _remove_records(indexed, diff['removed'], index_columns, strict=strict)
    _update_records(indexed, diff['changed'], strict=strict)


def _add_records(indexed, recs_to_add, index_columns, strict=True):
    indexed_to_add = records.index(recs_to_add, index_columns)
    for k, r in indexed_to_add.items():
        if strict and k in indexed:
            error.abort(
                'error: key {0} already exists in source document'.format(k)
            )
        indexed[k] = r


def _remove_records(indexed, recs_to_remove, index_columns, strict=True):
    indexed_to_remove = records.index(recs_to_remove, index_columns)
    for k, r in indexed_to_remove.items():
        if strict:
            v = indexed.get(k)
            if v is None:
                error.abort(
                    'ERROR: key {0} does not exist in source '
                    'document'.format(k)
                )
            if v != r:
                error.abort(
                    'ERROR: source document version of {0} has '
                    'changed'.format(k)
                )

        del indexed[k]


def _update_records(indexed, deltas, strict=True):
    for delta in deltas:
        k = tuple(delta['key'])
        field_changes = delta['fields']

        r = indexed.get(k)

        # what happens when the record is missing?
        if r is None:
            if strict:
                error.abort(
                    'ERROR: source document is missing record '
                    'for {0}'.format(k)
                )
            continue

        r = indexed[k]
        for field, from_to in field_changes.items():
            expected = from_to['from']
            if strict and r.get(field) != expected:
                error.abort(
                    'ERROR: source document version of {0} has '
                    'changed {1} field'.format(k, field)
                )
            r[field] = from_to['to']


def load(istream, strict=True):
    "Deserialize a patch object."
    # XXX validate it if strict
    return json.load(istream)


def save(diff, stream=sys.stdout, compact=False):
    "Serialize a patch object."
    if compact:
        json.dump(diff, stream)
    else:
        json.dump(diff, stream, indent=2, sort_keys=True)


def create(from_records, to_records, index_columns):
    """
    Diff two sets of records, using the index columns as the primary key for
    both datasets.
    """
    from_indexed = records.index(from_records, index_columns)
    to_indexed = records.index(to_records, index_columns)

    return create_indexed(from_indexed, to_indexed, index_columns)


def create_indexed(from_indexed, to_indexed, index_columns):
    # examine keys for overlap
    removed, added, shared = _compare_keys(from_indexed, to_indexed)

    # check for changed rows
    changed = _compare_rows(from_indexed, to_indexed, shared)

    diff = _assemble(removed, added, changed, from_indexed, to_indexed,
                     index_columns)

    return diff
<commit_msg>Add schema validation on patch load.<commit_after># -*- coding: utf-8 -*-
#
#  patch.py
#  csvdiff
#

"""
The the patch format.
"""

import sys
import json

import jsonschema

from . import records
from . import error


SCHEMA = {
    '$schema': 'http://json-schema.org/draft-04/schema#',
    'title': 'csvdiff',
    'description': 'The patch format used by csvdiff.',
    'type': 'object',
    'properties': {
        '_index': {
            'type': 'array',
            'minItems': 1,
            'items': {'type': 'string'},
        },
        'added': {
            'type': 'array',
            'items': {'type': 'object'},
        },
        'removed': {
            'type': 'array',
            'items': {'type': 'object'},
        },
        'changed': {
            'type': 'array',
            'items': {
                'type': 'object',
                'properties': {
                    'key': {'type': 'array',
                            'items': {'type': 'string'},
                            'minItems': 1},
                    'fields': {
                        'type': 'object',
                    },
                    'minProperties': 1,
                    'patternProperties': {
                        '.+': {'type': 'object',
                               'properties': {
                                   'from': {'type': 'string'},
                                   'to': {'type': 'string'},
                               },
                               'required': ['from', 'to']}
                    },
                },
            },
        },
    },
    'required': ['_index', 'added', 'changed', 'removed'],
}


def is_valid(diff):
    "Validate it against the schema."
    try:
        jsonschema.validate(diff, SCHEMA)
    except jsonschema.ValidationError:
        return False

    return True


def apply(diff, recs, strict=True):
    """
    Transform the records with the patch. May fail if the records do not
    match those expected in the patch.
    """
    index_columns = diff['_index']
    indexed = records.index(recs, index_columns, strict=strict)
    _add_records(indexed, diff['added'], index_columns, strict=strict)
    _remove_records(indexed, diff['removed'], index_columns, strict=strict)
    _update_records(indexed, diff['changed'], strict=strict)


def _add_records(indexed, recs_to_add, index_columns, strict=True):
    indexed_to_add = records.index(recs_to_add, index_columns)
    for k, r in indexed_to_add.items():
        if strict and k in indexed:
            error.abort(
                'error: key {0} already exists in source document'.format(k)
            )
        indexed[k] = r


def _remove_records(indexed, recs_to_remove, index_columns, strict=True):
    indexed_to_remove = records.index(recs_to_remove, index_columns)
    for k, r in indexed_to_remove.items():
        if strict:
            v = indexed.get(k)
            if v is None:
                error.abort(
                    'ERROR: key {0} does not exist in source '
                    'document'.format(k)
                )
            if v != r:
                error.abort(
                    'ERROR: source document version of {0} has '
                    'changed'.format(k)
                )

        del indexed[k]


def _update_records(indexed, deltas, strict=True):
    for delta in deltas:
        k = tuple(delta['key'])
        field_changes = delta['fields']

        r = indexed.get(k)

        # what happens when the record is missing?
        if r is None:
            if strict:
                error.abort(
                    'ERROR: source document is missing record '
                    'for {0}'.format(k)
                )
            continue

        r = indexed[k]
        for field, from_to in field_changes.items():
            expected = from_to['from']
            if strict and r.get(field) != expected:
                error.abort(
                    'ERROR: source document version of {0} has '
                    'changed {1} field'.format(k, field)
                )
            r[field] = from_to['to']


def load(istream, strict=True):
    "Deserialize a patch object."
    # XXX validate it if strict
    diff = json.load(istream)
    if strict:
        jsonschema.validate(diff, SCHEMA)
    return diff


def save(diff, stream=sys.stdout, compact=False):
    "Serialize a patch object."
    flags = ({}
             if compact
             else {'indent': 2, 'sort_keys': True})
    json.dump(diff, stream, **flags)


def create(from_records, to_records, index_columns):
    """
    Diff two sets of records, using the index columns as the primary key for
    both datasets.
    """
    from_indexed = records.index(from_records, index_columns)
    to_indexed = records.index(to_records, index_columns)

    return create_indexed(from_indexed, to_indexed, index_columns)


def create_indexed(from_indexed, to_indexed, index_columns):
    # examine keys for overlap
    removed, added, shared = _compare_keys(from_indexed, to_indexed)

    # check for changed rows
    changed = _compare_rows(from_indexed, to_indexed, shared)

    diff = _assemble(removed, added, changed, from_indexed, to_indexed,
                     index_columns)

    return diff

{'fraction_non_alphanumeric': 0.12062408581179912, 'fraction_numerical': 0.0025353486104339345, 'mean_word_length': 1.6989473684210525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '715998', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2861, 'n_words': 765}