<commit_before>
from __future__ import absolute_import

import collections
from itertools import chain
import re
import types
import unicodedata

from richenum import OrderedRichEnum
from richenum import OrderedRichEnumValue

from normalize.property import SafeProperty
from normalize.coll import Collection
from normalize.coll import ListCollection
import normalize.exc as exc
from normalize.record import Record
from normalize.record import record_id
from normalize.selector import FieldSelector


class DiffTypes(OrderedRichEnum):
    """
    An "enum" to represent types of diffs
    """
    class EnumValue(OrderedRichEnumValue):
        pass

    NO_CHANGE = EnumValue(1, "none", "UNCHANGED")
    ADDED = EnumValue(2, "added", "ADDED")
    REMOVED = EnumValue(3, "removed", "REMOVED")
    MODIFIED = EnumValue(4, "modified", "MODIFIED")


def _coerce_diff(dt):
    if not isinstance(dt, OrderedRichEnumValue):
        if isinstance(dt, (int, long)):
            dt = DiffTypes.from_index(dt)
        else:
            dt = DiffTypes.from_canonical(dt)
    return dt


class DiffInfo(Record):
    """
    Container for storing diff information that can be used to reconstruct the
    values diffed.
    """
    diff_type = SafeProperty(
        coerce=_coerce_diff,
        isa=DiffTypes.EnumValue,
        required=True,
    )
    base = SafeProperty(isa=FieldSelector, required=True)
    other = SafeProperty(isa=FieldSelector, required=True)

    def __str__(self):
        if self.base.path != self.other.path:
            pathinfo = (
                self.base.path if (
                    len(self.base) > len(self.other) and
                    self.base.startswith(self.other)
                ) else self.other.path if (
                    len(self.other) > len(self.base) and
                    self.other.startswith(self.base)
                ) else "(%s/%s)" % (self.base.path, self.other.path)
            )
        else:
            pathinfo = self.other.path
        difftype = self.diff_type.display_name
        return "<DiffInfo: %s %s>" % (difftype, pathinfo)


class _Nothing(object):
    def __repr__(self):
        return "(not set)"


_nothing = _Nothing()


class DiffOptions(object):
    """Optional data structure to pass diff options down"""
    def __init__(self, ignore_ws=True, ignore_case=False,
                 unicode_normal=True, unchanged=False,
                 ignore_empty_slots=False,
                 duck_type=False, extraneous=False):
        self.ignore_ws = ignore_ws
        self.ignore_case = ignore_case
        self.ignore_empty_slots = ignore_empty_slots
        self.unicode_normal = unicode_normal
        self.unchanged = unchanged
        self.duck_type = duck_type
        self.extraneous = extraneous

    def items_equal(self, a, b):
        return self.normalize_val(a) == self.normalize_val(b)

    def normalize_whitespace(self, value):
        if isinstance(value, unicode):
            return u" ".join(
                x for x in re.split(r'\s+', value, flags=re.UNICODE) if
                len(x)
            )
        else:
            return " ".join(value.split())

    def normalize_unf(self, value):
        if isinstance(value, unicode):
            return unicodedata.normalize('NFC', value)
        else:
            return value

    def normalize_case(self, value):
        # FIXME: this will do the wrong thing for letters in some languages, eg
        # Greek, Turkish.  Correct, locale-dependent unicode case folding is
        # left as an exercise for a subclass.
        return value.upper()

    def value_is_empty(self, value):
        return (not value and isinstance(value, (basestring, types.NoneType)))

    def normalize_val(self, value=_nothing):
        if isinstance(value, basestring):
            if self.ignore_ws:
                value = self.normalize_whitespace(value)
            if self.ignore_case:
                value = self.normalize_case(value)
            if self.unicode_normal:
                value = self.normalize_unf(value)
        elif self.ignore_empty_slots and self.value_is_empty(value):
            value = _nothing
        return value

    def record_id(self, record, type_=None):
        """Retrieve an object identifier from the given record; if it is an
        alien class, and the type is provided, then use duck typing to get the
        corresponding fields of the alien class."""
        return getattr(record, "__pk__", record_id(record, type_))


def compare_record_iter(a, b, fs_a=None, fs_b=None, options=None):
    if not options:
        options = DiffOptions()

    if not options.duck_type and type(a) != type(b):
        raise TypeError(
            "cannot compare %s with %s" % (type(a).__name__, type(b).__name__)
        )<commit_msg>diff: don't call record_id() if the class implements __pk__<commit_after>
from __future__ import absolute_import

import collections
from itertools import chain
import re
import types
import unicodedata

from richenum import OrderedRichEnum
from richenum import OrderedRichEnumValue

from normalize.property import SafeProperty
from normalize.coll import Collection
from normalize.coll import ListCollection
import normalize.exc as exc
from normalize.record import Record
from normalize.record import record_id
from normalize.selector import FieldSelector


class DiffTypes(OrderedRichEnum):
    """
    An "enum" to represent types of diffs
    """
    class EnumValue(OrderedRichEnumValue):
        pass

    NO_CHANGE = EnumValue(1, "none", "UNCHANGED")
    ADDED = EnumValue(2, "added", "ADDED")
    REMOVED = EnumValue(3, "removed", "REMOVED")
    MODIFIED = EnumValue(4, "modified", "MODIFIED")


def _coerce_diff(dt):
    if not isinstance(dt, OrderedRichEnumValue):
        if isinstance(dt, (int, long)):
            dt = DiffTypes.from_index(dt)
        else:
            dt = DiffTypes.from_canonical(dt)
    return dt


class DiffInfo(Record):
    """
    Container for storing diff information that can be used to reconstruct the
    values diffed.
    """
    diff_type = SafeProperty(
        coerce=_coerce_diff,
        isa=DiffTypes.EnumValue,
        required=True,
    )
    base = SafeProperty(isa=FieldSelector, required=True)
    other = SafeProperty(isa=FieldSelector, required=True)

    def __str__(self):
        if self.base.path != self.other.path:
            pathinfo = (
                self.base.path if (
                    len(self.base) > len(self.other) and
                    self.base.startswith(self.other)
                ) else self.other.path if (
                    len(self.other) > len(self.base) and
                    self.other.startswith(self.base)
                ) else "(%s/%s)" % (self.base.path, self.other.path)
            )
        else:
            pathinfo = self.other.path
        difftype = self.diff_type.display_name
        return "<DiffInfo: %s %s>" % (difftype, pathinfo)


class _Nothing(object):
    def __repr__(self):
        return "(not set)"


_nothing = _Nothing()


class DiffOptions(object):
    """Optional data structure to pass diff options down"""
    def __init__(self, ignore_ws=True, ignore_case=False,
                 unicode_normal=True, unchanged=False,
                 ignore_empty_slots=False,
                 duck_type=False, extraneous=False):
        self.ignore_ws = ignore_ws
        self.ignore_case = ignore_case
        self.ignore_empty_slots = ignore_empty_slots
        self.unicode_normal = unicode_normal
        self.unchanged = unchanged
        self.duck_type = duck_type
        self.extraneous = extraneous

    def items_equal(self, a, b):
        return self.normalize_val(a) == self.normalize_val(b)

    def normalize_whitespace(self, value):
        if isinstance(value, unicode):
            return u" ".join(
                x for x in re.split(r'\s+', value, flags=re.UNICODE) if
                len(x)
            )
        else:
            return " ".join(value.split())

    def normalize_unf(self, value):
        if isinstance(value, unicode):
            return unicodedata.normalize('NFC', value)
        else:
            return value

    def normalize_case(self, value):
        # FIXME: this will do the wrong thing for letters in some languages, eg
        # Greek, Turkish.  Correct, locale-dependent unicode case folding is
        # left as an exercise for a subclass.
        return value.upper()

    def value_is_empty(self, value):
        return (not value and isinstance(value, (basestring, types.NoneType)))

    def normalize_val(self, value=_nothing):
        if isinstance(value, basestring):
            if self.ignore_ws:
                value = self.normalize_whitespace(value)
            if self.ignore_case:
                value = self.normalize_case(value)
            if self.unicode_normal:
                value = self.normalize_unf(value)
        elif self.ignore_empty_slots and self.value_is_empty(value):
            value = _nothing
        return value

    def record_id(self, record, type_=None):
        """Retrieve an object identifier from the given record; if it is an
        alien class, and the type is provided, then use duck typing to get the
        corresponding fields of the alien class."""
        pk = getattr(record, "__pk__", None)
        return pk if pk is not None else record_id(record, type_)


def compare_record_iter(a, b, fs_a=None, fs_b=None, options=None):
    if not options:
        options = DiffOptions()

    if not options.duck_type and type(a) != type(b):
        raise TypeError(
            "cannot compare %s with %s" % (type(a).__name__, type(b).__name__)
        )
{'fraction_non_alphanumeric': 0.09065478657273104, 'fraction_numerical': 0.0008288437629506838, 'mean_word_length': 2.434009249377446, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1953278', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2734, 'n_words': 773}