<commit_before># -*- coding: utf-8 -*-
import sys
import pprint
import six
from collections import Mapping, MutableMapping, namedtuple


PY3 = sys.version_info[0] == 3

dict_setitem = dict.__setitem__
dict_getitem = dict.__getitem__
dict_delitem = dict.__delitem__
dict_contains = dict.__contains__
dict_clear = dict.clear


class MappingView(object):
    def __init__(self, mapping):
        self._mapping = mapping

    def __len__(self):
        return len(self._mapping)

    def __repr__(self):
        return '%s(%r)' % (type(self).__name__, self._mapping)


class KeysView(MappingView):
    def __contains__(self, key):
        return key in self._mapping

    def __iter__(self):
        for key, _ in self._mapping:
            yield key


class ItemsView(MappingView):
    def __contains__(self, item):
        key, value = item
        return value in self._mapping.getlist(key)

    def __iter__(self):
        for item in self._mapping:
            yield item


class ValuesView(MappingView):
    def __contains__(self, value):
        for _, v in self._mapping:
            if v == value:
                return True
        return False

    def __iter__(self):
        for _, value in self._mapping:
            yield value


class OrderedMultiDict(dict, MutableMapping):
    """A ``dict`` like container.

    This container preserves the original ordering as well as allows multiple
    values for the same key. It provides a similar similar semantics to a
    ``list`` of ``tuples`` but with ``dict`` style access.

    Using ``__setitem__`` syntax overwrites all fields with the same key and
    ``__getitem__`` will return the first value with the key.
    """

    def __init__(self, *args, **kwargs):
        self.__items = []
        self.extend(*args, **kwargs)

    def __setitem__(self, key, value):
        if key not in self:
            return self.append(key, value)

        dict_setitem(self, key, [value])
        iteritems = iter(self.__items)

        for index, (old_key, old_value) in enumerate(iteritems):
            if old_key == key:
                # replace first occurrence
                self.__items[index] = (key, value)
                break

        tail = [item for item in iteritems if item[0] != key]
        self.__items[index + 1:] = tail

    def __getitem__(self, key):
        return dict_getitem(self, key)[0]

    def __delitem__(self, key):
        dict_delitem(self, key)
        self.__items = [item for item in self.__items if item[0] != key]

    def __iter__(self):
        return iter(self.__items)

    def __len__(self):
        return len(self.__items)

    def __eq__(self, other):
        if not isinstance(other, type(self)):
            return False

        if len(self) != len(other):
            return False

        items1 = six.iteritems(self)
        items2 = six.iteritems(other)

        for ((key1, value1), (key2, value2)) in six.moves.zip(items1, items2):
            if key1 != key2:
                return False

            if value1 != value2:
                return False

        return True

    def __repr__(self):
        if not self.__items:
            return '%s([])' % type(self).__name__

        lines = []
        for item in self.__items:
            for line in pprint.pformat(item).splitlines():
                lines.append('  ' + line)

        return "%s([\n%s\n])" % (type(self).__name__, '\n'.join(lines))

    get = MutableMapping.get
    update = MutableMapping.update
    pop = MutableMapping.pop
    popitem = MutableMapping.popitem<commit_msg>Fix implementation of __ne__.<commit_after># -*- coding: utf-8 -*-
import sys
import pprint
import six
from collections import Mapping, MutableMapping, namedtuple


PY3 = sys.version_info[0] == 3

dict_setitem = dict.__setitem__
dict_getitem = dict.__getitem__
dict_delitem = dict.__delitem__
dict_contains = dict.__contains__
dict_clear = dict.clear


class MappingView(object):
    def __init__(self, mapping):
        self._mapping = mapping

    def __len__(self):
        return len(self._mapping)

    def __repr__(self):
        return '%s(%r)' % (type(self).__name__, self._mapping)


class KeysView(MappingView):
    def __contains__(self, key):
        return key in self._mapping

    def __iter__(self):
        for key, _ in self._mapping:
            yield key


class ItemsView(MappingView):
    def __contains__(self, item):
        key, value = item
        return value in self._mapping.getlist(key)

    def __iter__(self):
        for item in self._mapping:
            yield item


class ValuesView(MappingView):
    def __contains__(self, value):
        for _, v in self._mapping:
            if v == value:
                return True
        return False

    def __iter__(self):
        for _, value in self._mapping:
            yield value


class OrderedMultiDict(dict, MutableMapping):
    """A ``dict`` like container.

    This container preserves the original ordering as well as allows multiple
    values for the same key. It provides a similar similar semantics to a
    ``list`` of ``tuples`` but with ``dict`` style access.

    Using ``__setitem__`` syntax overwrites all fields with the same key and
    ``__getitem__`` will return the first value with the key.
    """

    def __init__(self, *args, **kwargs):
        self.__items = []
        self.extend(*args, **kwargs)

    def __setitem__(self, key, value):
        if key not in self:
            return self.append(key, value)

        dict_setitem(self, key, [value])
        iteritems = iter(self.__items)

        for index, (old_key, old_value) in enumerate(iteritems):
            if old_key == key:
                # replace first occurrence
                self.__items[index] = (key, value)
                break

        tail = [item for item in iteritems if item[0] != key]
        self.__items[index + 1:] = tail

    def __getitem__(self, key):
        return dict_getitem(self, key)[0]

    def __delitem__(self, key):
        dict_delitem(self, key)
        self.__items = [item for item in self.__items if item[0] != key]

    def __iter__(self):
        return iter(self.__items)

    def __len__(self):
        return len(self.__items)

    def __eq__(self, other):
        if not isinstance(other, type(self)):
            return False

        if len(self) != len(other):
            return False

        items1 = six.iteritems(self)
        items2 = six.iteritems(other)

        for ((key1, value1), (key2, value2)) in six.moves.zip(items1, items2):
            if key1 != key2:
                return False

            if value1 != value2:
                return False

        return True

    def __ne__(self, other):
        return not (self == other)

    def __repr__(self):
        if not self.__items:
            return '%s([])' % type(self).__name__

        lines = []
        for item in self.__items:
            for line in pprint.pformat(item).splitlines():
                lines.append('  ' + line)

        return "%s([\n%s\n])" % (type(self).__name__, '\n'.join(lines))

    get = MutableMapping.get
    update = MutableMapping.update
    pop = MutableMapping.pop
    popitem = MutableMapping.popitem
{'fraction_non_alphanumeric': 0.10376441172385054, 'fraction_numerical': 0.005556327267676066, 'mean_word_length': 2.257918552036199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3056160', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2044, 'n_words': 637}