<commit_before># -*- coding: utf-8 -*-

#    Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import itertools

import six


# Used for values that don't matter in sets backed by dicts...
_sentinel = object()


def _merge_in(target, iterable=None, sentinel=_sentinel):
    """Merges iterable into the target and returns the target."""
    if iterable is not None:
        for value in iterable:
            target.setdefault(value, sentinel)
    return target


class OrderedSet(collections.Set, collections.Hashable):
    """A read-only hashable set that retains insertion/initial ordering.

    It should work in all existing places that ``frozenset`` is used.

    See: https://mail.python.org/pipermail/python-ideas/2009-May/004567.html
    for an idea thread that *may* eventually (*someday*) result in this (or
    similar) code being included in the mainline python codebase (although
    the end result of that thread is somewhat discouraging in that regard).
    """
<commit_msg>Handle collections.abc deprecations<commit_after># -*- coding: utf-8 -*-

#    Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections

# TODO(smcginnis) update this once six has support for collections.abc
# (https://github.com/benjaminp/six/pull/241) or clean up once we drop py2.7.
try:
    from collections.abc import Hashable
    from collections.abc import Set
except ImportError:
    from collections import Hashable
    from collections import Set

import itertools

import six


# Used for values that don't matter in sets backed by dicts...
_sentinel = object()


def _merge_in(target, iterable=None, sentinel=_sentinel):
    """Merges iterable into the target and returns the target."""
    if iterable is not None:
        for value in iterable:
            target.setdefault(value, sentinel)
    return target


class OrderedSet(Set, Hashable):
    """A read-only hashable set that retains insertion/initial ordering.

    It should work in all existing places that ``frozenset`` is used.

    See: https://mail.python.org/pipermail/python-ideas/2009-May/004567.html
    for an idea thread that *may* eventually (*someday*) result in this (or
    similar) code being included in the mainline python codebase (although
    the end result of that thread is somewhat discouraging in that regard).
    """

{'fraction_non_alphanumeric': 0.07793696275071633, 'fraction_numerical': 0.012320916905444125, 'mean_word_length': 3.44147582697201, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722343', 'n_tokens_mistral': 976, 'n_tokens_neox': 929, 'n_words': 422}