<commit_before>from __future__ import absolute_import
import inspect
import operator
import collections
from itertools import count

from .util import clean_params, collect_doc_classes
from .types import instantiate, Type
from .compat import string_types


class Expression(object):
    def _collect_doc_classes(self):
        return set()

    def compile(self, compiler):
        return compiler.compiled_expression(self)

    def to_elastic(self, compiler):
        return self.compile(compiler).body

    to_dict = to_elastic


class Literal(Expression):
    __visit_name__ = 'literal'

    def __init__(self, obj):
        self.obj = obj


class Params(Expression, collections.Mapping):
    __visit_name__ = 'params'

    def __init__(self, *args, **kwargs):
        params = {}
        for d in args:
            params.update(d)
        params.update(kwargs)
        self._params = {}
        for k, v in clean_params(params).items():
            if k.endswith('_') and not k.startswith('_'):
                k = k.rstrip('_')
            self._params[k] = v

    def _collect_doc_classes(self):
        return collect_doc_classes(self._params)

    def __len__(self):
        return len(self._params)
<commit_msg>Fix Mapping import for compat with Python 3.10<commit_after>from __future__ import absolute_import
import inspect
import operator
import collections
from itertools import count

from .util import clean_params, collect_doc_classes
from .types import instantiate, Type
from .compat import string_types, Mapping


class Expression(object):
    def _collect_doc_classes(self):
        return set()

    def compile(self, compiler):
        return compiler.compiled_expression(self)

    def to_elastic(self, compiler):
        return self.compile(compiler).body

    to_dict = to_elastic


class Literal(Expression):
    __visit_name__ = 'literal'

    def __init__(self, obj):
        self.obj = obj


class Params(Expression, Mapping):
    __visit_name__ = 'params'

    def __init__(self, *args, **kwargs):
        params = {}
        for d in args:
            params.update(d)
        params.update(kwargs)
        self._params = {}
        for k, v in clean_params(params).items():
            if k.endswith('_') and not k.startswith('_'):
                k = k.rstrip('_')
            self._params[k] = v

    def _collect_doc_classes(self):
        return collect_doc_classes(self._params)

    def __len__(self):
        return len(self._params)

{'fraction_non_alphanumeric': 0.09263754045307443, 'fraction_numerical': 0.0012135922330097086, 'mean_word_length': 2.858034321372855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4409224', 'n_tokens_mistral': 735, 'n_tokens_neox': 702, 'n_words': 196}