<commit_before>import uuid
import warnings
from enum import Enum, auto, unique

from ocdsmerge.errors import DuplicateIdValueWarning


@unique
class MergeStrategy(Enum):
    APPEND = auto()
    MERGE_BY_POSITION = auto()


globals().update(MergeStrategy.__members__)


class IdValue(str):
    """
    A string with ``identifier`` and ``original_value`` properties.
    """
    def __init__(self, identifier):
        self.identifier = identifier
        str.__init__(identifier)

    @property
    def original_value(self):
        return self._original_value

    @original_value.setter
    def original_value(self, original_value):
        self._original_value = original_value


class IdDict(dict):
    """
    A dictionary with an ``identifier`` property.
    """
    @property
    def identifier(self):
        return self._identifier

    @identifier.setter
    def identifier(self, identifier):
        self._identifier = identifier


def is_versioned_value(value):
    """
    Returns whether the value is a versioned value.
    """
    return {'releaseID', 'releaseDate', 'releaseTag', 'value'} == set(value)


def _path_without_array_indexes(path):
    return tuple(part for part in path if type(part) is not int)


def flatten(obj, merge_rules, path=None, flattened=None):
    """
    Flattens a JSON object into key-value pairs, in which the key is the JSON path as a tuple. For example:

    .. code:: json

       {
           "a": "I am a",
           "b": ["A", "list"],
           "c": [
               {"ca": "I am ca"},
               {"cb": "I am cb"}
           ]
       }

    flattens to:

    .. code:: python

       {
           ('a',): 'I am a',
           ('b',): ['A', 'list'],
           ('c', 0, 'ca'): 'I am ca',
           ('c', 1, 'cb'): 'I am cb',
       }
    """
    if path is None:
        path = ()
    if flattened is None:
        flattened = {}

    if isinstance(obj, dict):
        iterable = obj.items()
        if not iterable:
            flattened[path] = {}
    else:
        iterable = enumerate(obj)
        if not iterable:
            flattened[path] = []

    for key, value in iterable:
        new_path = path + (key,)
        # Remove array indices to find the merge rule for this JSON path in the data.
        new_path_merge_rules = merge_rules.get(_path_without_array_indexes(new_path), [])

        if 'omitWhenMerged' in new_path_merge_rules:
            continue
        # If it's neither an object nor an array, if it's `wholeListMerge`, if it's an array containing non-objects
        # (even if `wholeListMerge` is `false`), or if it's versioned values, use the whole list merge strategy.
        # Note: Behavior is undefined and inconsistent if the array is not in the schema and contains objects in some
        # cases but not in others.
        # See https://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
        # See https://standard.open-contracting.org/1.1-dev/en/schema/merging/#objects
        elif not isinstance(value, (dict, list)) or 'wholeListMerge' in new_path_merge_rules or \
                isinstance(value, list) and any(not isinstance(item, dict) for item in value) or \
                len(value) and all(is_versioned_value(item) for item in value):
            flattened[new_path] = value
        # Recurse into non-empty objects, and arrays of objects that aren't `wholeListMerge`.
        elif value:
            flatten(value, merge_rules, new_path, flattened)

    return flattened


def process_flattened(flattened, rule_overrides):
    """
    Replace numbers in JSON paths (representing positions in arrays) with special objects. This ensures that objects
    in arrays with different `id` values have different JSON paths – and makes it easy to identify such arrays.
    """
    # Keep arrays in order.
    processed = {}

    # Cache identifiers, to avoid minting a new ID for each field of the same object.
    # e.g. {('awards', 0): '1', ('awards', 1): '2'}
    # This also tracks all identifiers of objects in each array, to warn about collisions.
    # e.g. {('awards',): {'1': 0, '2': 1}}
    identifiers = {}

    for key in flattened:
        new_key = []
        for end, part in enumerate(key, 1):
            # If this is a path to an item in an array.
            if type(part) is int:
                path = key[:end]
                if path in identifiers:
                    part = identifiers[path]
                else:
                    scope = path[:-1]

                    # If it is an array of objects, get the `id` value to apply the identifier merge strategy.
                    # https://standard.open-contracting.org/latest/en/schema/merging/#identifier-merge
                    id_value = flattened.get(path + ('id',))

                    # If the object contained no top-level `id` value, set a unique value.
                    if id_value is None:
                        identifier = str(uuid.uuid4())  # uuid's equality test is very slow
                    else:
                        identifier = id_value

                    # Keep the integer index and default `part` for the warning.
                    index = part
                    default = IdValue(identifier)

                    rule_path = _path_without_array_indexes(scope)
                    rule = rule_overrides.get(rule_path)
                    if rule == MergeStrategy.APPEND:
                        part = IdValue(uuid.uuid4())
                    elif rule == MergeStrategy.MERGE_BY_POSITION:
                        part = IdValue(index)
                    else:
                        part = default

                    # Save the original value. (If the value is an integer, this avoids coercing it to a string.)
                    part.original_value = id_value

                    # Check whether the value is repeated in other objects in the array.
                    if scope not in identifiers:
                        identifiers[scope] = {}
                    if default not in identifiers[scope]:
                        identifiers[scope][default] = index
                    elif identifiers[scope][default] != index:<commit_msg>Improve performance by building merge_rule_path instead of re-calculating it. Make logic in process_flattened clearer.<commit_after>import uuid
import warnings
from enum import Enum, auto, unique

from ocdsmerge.errors import DuplicateIdValueWarning


@unique
class MergeStrategy(Enum):
    APPEND = auto()
    MERGE_BY_POSITION = auto()


globals().update(MergeStrategy.__members__)


class IdValue(str):
    """
    A string with ``identifier`` and ``original_value`` properties.
    """
    def __init__(self, identifier):
        self.identifier = identifier
        str.__init__(identifier)

    @property
    def original_value(self):
        return self._original_value

    @original_value.setter
    def original_value(self, original_value):
        self._original_value = original_value


class IdDict(dict):
    """
    A dictionary with an ``identifier`` property.
    """
    @property
    def identifier(self):
        return self._identifier

    @identifier.setter
    def identifier(self, identifier):
        self._identifier = identifier


def is_versioned_value(value):
    """
    Returns whether the value is a versioned value.
    """
    return {'releaseID', 'releaseDate', 'releaseTag', 'value'} == set(value)


def flatten(obj, merge_rules, path=None, flattened=None, merge_rule_path=None):
    """
    Flattens a JSON object into key-value pairs, in which the key is the JSON path as a tuple. For example:

    .. code:: json

       {
           "a": "I am a",
           "b": ["A", "list"],
           "c": [
               {"ca": "I am ca"},
               {"cb": "I am cb"}
           ]
       }

    flattens to:

    .. code:: python

       {
           ('a',): 'I am a',
           ('b',): ['A', 'list'],
           ('c', 0, 'ca'): 'I am ca',
           ('c', 1, 'cb'): 'I am cb',
       }
    """
    if path is None:
        path = ()
    if flattened is None:
        flattened = {}
    if merge_rule_path is None:
        merge_rule_path = ()

    if isinstance(obj, dict):
        iterable = obj.items()
        if not iterable:
            flattened[path] = {}
    else:
        iterable = enumerate(obj)
        if not iterable:
            flattened[path] = []

    for key, value in iterable:
        new_path = path + (key,)
        if type(key) is int:
            new_merge_rule_path = merge_rule_path
        else:
            new_merge_rule_path = merge_rule_path + (key,)

        # Remove array indices to find the merge rule for this JSON path in the data.
        new_path_merge_rules = merge_rules.get(new_merge_rule_path, set())

        if 'omitWhenMerged' in new_path_merge_rules:
            continue
        # If it's neither an object nor an array, if it's `wholeListMerge`, if it's an array containing non-objects
        # (even if `wholeListMerge` is `false`), or if it's versioned values, use the whole list merge strategy.
        # Note: Behavior is undefined and inconsistent if the array is not in the schema and contains objects in some
        # cases but not in others.
        # See https://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
        # See https://standard.open-contracting.org/1.1-dev/en/schema/merging/#objects
        elif not isinstance(value, (dict, list)) or 'wholeListMerge' in new_path_merge_rules or \
                type(value) is list and any(not isinstance(item, dict) for item in value) or \
                len(value) and all(is_versioned_value(item) for item in value):
            flattened[new_path] = value
        # Recurse into non-empty objects, and arrays of objects that aren't `wholeListMerge`.
        elif value:
            flatten(value, merge_rules, new_path, flattened, new_merge_rule_path)

    return flattened


def process_flattened(flattened, rule_overrides):
    """
    Replace numbers in JSON paths (representing positions in arrays) with special objects. This ensures that objects
    in arrays with different `id` values have different JSON paths – and makes it easy to identify such arrays.
    """
    # Keep arrays in order.
    processed = {}

    # Cache identifiers, to avoid minting a new ID for each field of the same object.
    # e.g. {('awards', 0): '1', ('awards', 1): '2'}
    # This also tracks all identifiers of objects in each array, to warn about collisions.
    # e.g. {('awards',): {'1': 0, '2': 1}}
    identifiers = {}

    for key in flattened:
        new_key = []
        for end, part in enumerate(key, 1):
            # If this is a path to an item in an array.
            if type(part) is int:
                path = key[:end]
                if path in identifiers:
                    part = identifiers[path]
                else:
                    scope = path[:-1]

                    # If it is an array of objects, get the `id` value to apply the identifier merge strategy.
                    # https://standard.open-contracting.org/latest/en/schema/merging/#identifier-merge
                    id_key = path + ('id',)
                    if id_key in flattened:
                        id_value = flattened[id_key]
                        identifier = id_value
                    # If the object contained no top-level `id` value, set a unique value.
                    else:
                        id_value = None
                        identifier = str(uuid.uuid4())  # uuid's equality test is very slow

                    # Keep the integer index and default `part` for the warning.
                    index = part
                    default = IdValue(identifier)

                    rule_path = tuple(p for p in scope if type(p) is not int)
                    rule = rule_overrides.get(rule_path)
                    if rule == MergeStrategy.APPEND:
                        part = IdValue(uuid.uuid4())
                    elif rule == MergeStrategy.MERGE_BY_POSITION:
                        part = IdValue(index)
                    else:
                        part = default

                    # Save the original value. (If the value is an integer, this avoids coercing it to a string.)
                    part.original_value = id_value

                    # Check whether the value is repeated in other objects in the array.
                    if scope not in identifiers:
                        identifiers[scope] = {}
                    if default not in identifiers[scope]:
                        identifiers[scope][default] = index
                    elif identifiers[scope][default] != index:
{'fraction_non_alphanumeric': 0.09539163258886442, 'fraction_numerical': 0.0028310789556464295, 'mean_word_length': 1.916074294886494, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1417281', 'n_tokens_mistral': 3574, 'n_tokens_neox': 3347, 'n_words': 1185}