<commit_before>import operator

import isodate

from django import forms
from django.utils.timezone import utc
from django.utils.encoding import smart_str

from crashstats.crashstats.utils import parse_isodate


OPERATORS = (
    '__true__', '__null__', '$', '~', '^', '@', '=', '<=', '>=', '<', '>',
    '!__true__', '!__null__', '!$', '!~', '!^', '!@', '!=', '!'
)


def split_on_operator(value):
    for op in sorted(OPERATORS, key=len, reverse=True):
        if value.startswith(op):
            value = value[len(op):]
            return (op, value)

    return (None, value)


class PrefixedField(object):
    """Special field that accepts an operator as prefix in the value.

    Removes the prefix from the initial value before the validation process
    starts, and put it back in a different attribute once the validation
    process is finished. The cleaned value is the one without the prefix, thus
    allowing to use the real value and check its type.

    The validated, prefixed value is available in `prefixed_value` as a string,
    and the prefix is in `operator`.

    This is needed to allow fields like IntegerField to accept values
    containing an operator. For example, a value such as '>13' will raise
    a ValidationError in the basic django IntegerField. Using a PrefixedField
    based IntegerField, this value is perfectly valid.

    """

    operator = None
    prefixed_value = None

    def to_python(self, value):
        if isinstance(value, str):
            self.operator, value = split_on_operator(value)

        return super(PrefixedField, self).to_python(value)
<commit_msg>fix bug 1460707: fixed python 3 lint errors<commit_after>import operator

import isodate
from six import string_types

from django import forms
from django.utils.timezone import utc
from django.utils.encoding import smart_str

from crashstats.crashstats.utils import parse_isodate


OPERATORS = (
    '__true__', '__null__', '$', '~', '^', '@', '=', '<=', '>=', '<', '>',
    '!__true__', '!__null__', '!$', '!~', '!^', '!@', '!=', '!'
)


def split_on_operator(value):
    for op in sorted(OPERATORS, key=len, reverse=True):
        if value.startswith(op):
            value = value[len(op):]
            return (op, value)

    return (None, value)


class PrefixedField(object):
    """Special field that accepts an operator as prefix in the value.

    Removes the prefix from the initial value before the validation process
    starts, and put it back in a different attribute once the validation
    process is finished. The cleaned value is the one without the prefix, thus
    allowing to use the real value and check its type.

    The validated, prefixed value is available in `prefixed_value` as a string,
    and the prefix is in `operator`.

    This is needed to allow fields like IntegerField to accept values
    containing an operator. For example, a value such as '>13' will raise
    a ValidationError in the basic django IntegerField. Using a PrefixedField
    based IntegerField, this value is perfectly valid.

    """

    operator = None
    prefixed_value = None

    def to_python(self, value):
        if isinstance(value, string_types):
            self.operator, value = split_on_operator(value)

        return super(PrefixedField, self).to_python(value)

{'fraction_non_alphanumeric': 0.10199757869249394, 'fraction_numerical': 0.0036319612590799033, 'mean_word_length': 3.43029490616622, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4038055', 'n_tokens_mistral': 967, 'n_tokens_neox': 934, 'n_words': 361}