<commit_before>from   six       import itervalues, string_types
from   .         import errors
from   .normdict import NormalizedDict
from   .scanner  import scan_file, scan_lines, scan_string
from   .types    import lower
from   .util     import unfold

class HeaderParser(object):
    """
    A parser for RFC 822-style header sections.  Define the fields the parser
    should recognize with the `add_field` method, configure handling of
    unrecognized fields with `add_additional`, and then parse input with
    `parse_file` or `parse_string`.

    :param callable normalizer: a callable to apply to field names in order to
        normalize them for storage & comparison; default:
        ``operator.methodcaller('lower')``
    :param bool body: whether the parser should allow or forbid a body after
        the header section; `True` means a body is required, `False` means a
        body is prohibited, and `None` means a body is optional
    """

    def __init__(self, normalizer=None, body=None):
        self.normalizer = normalizer or lower
        self.body = body
        self.fielddefs = dict()
        self.dests = set()
        self.additional = None
        self.custom_dests = False

    def add_field(self, name, *altnames, **kwargs):
        kwargs.setdefault('dest', name)
        hd = NamedField(name=name, **kwargs)
        normed = set(map(self.normalizer, (name,) + altnames))
        # Error before modifying anything:
        redefs = [n for n in self.fielddefs if n in normed]
        if redefs:
            raise ValueError('field defined more than once: ' + repr(redefs[0]))
        if self.normalizer(hd.dest) in self.dests:
            raise ValueError('destination defined more than once: '
                             + repr(hd.dest))
        if self.normalizer(hd.dest) not in normed:
            if self.additional is not None:
                raise ValueError('add_additional and `dest` are mutually exclusive')
            self.custom_dests = True
        for n in normed:
            self.fielddefs[n] = hd
        self.dests.add(self.normalizer(hd.dest))

    def add_additional(self, allow=True, **kwargs):
        if allow:
            if self.custom_dests:
                raise ValueError('add_additional and `dest` are mutually exclusive')
            self.additional = FieldDef(**kwargs)<commit_msg>Docstring for `add_field`<commit_after>from   six       import itervalues, string_types
from   .         import errors
from   .normdict import NormalizedDict
from   .scanner  import scan_file, scan_lines, scan_string
from   .types    import lower
from   .util     import unfold

class HeaderParser(object):
    """
    A parser for RFC 822-style header sections.  Define the fields the parser
    should recognize with the `add_field` method, configure handling of
    unrecognized fields with `add_additional`, and then parse input with
    `parse_file` or `parse_string`.

    :param callable normalizer: By default, the parser will consider two field
        names to be equal iff their lowercased forms are equal.  This can be
        overridden by setting ``normalizer`` to a custom callable that takes a
        field name and returns a "normalized" name for use in equality testing.
        The normalizer will also be used when looking up keys in the
        `NormalizedDict` instances returned by the parser's ``parse_*``
        methods.

    :param bool body: whether the parser should allow or forbid a body after
        the header section; `True` means a body is required, `False` means a
        body is prohibited, and `None` means a body is optional
    """

    def __init__(self, normalizer=None, body=None):
        self.normalizer = normalizer or lower
        self.body = body
        self.fielddefs = dict()
        self.dests = set()
        self.additional = None
        self.custom_dests = False

    def add_field(self, name, *altnames, **kwargs):
        """
        Define a header field for the parser to parse.  During parsing, if a
        field is encountered whose name (*modulo* normalization) equals
        ``name`` or is in ``altnames``, the field's value will be processed
        according to the options in ``**kwargs``.  (If no options are
        specified, the value will just be stored in the result dictionary.)

        :param string name: the primary name for the field, used in error
            messages and as the default value of ``dest``

        :param strings altnames: field name synonyms

        :param dest: The key in the result dictionary in which the field's
            value(s) will be stored; defaults to ``name``.  When additional
            headers are enabled (see `add_additional`), ``dest`` can only equal
            one of the field's names.

        :param bool required: If `True` (default `False`), the ``parse_*``
            methods will raise a `~headerparser.errors.MissingFieldError` if
            the field is not present in the input

        :param default: The value to associate with the field if it is not
            present in the input.  If no default value is specified, the field
            will be omitted from the result dictionary if it is not present in
            the input.  ``default`` cannot be set when the field is required.
            ``type`` and ``unfold`` will not be applied to the default value,
            and the default value need not belong to ``choices``.

        :param bool unfold: If `True` (default `False`), the field value will
            be "unfolded" (i.e., line breaks will be removed and whitespace
            around line breaks will be converted to a single space) before
            applying ``type``

        :param callable type: a callable to apply to the field value before
            storing it in the result dictionary

        :param iterable choices: A sequence of values which the field is
            allowed to have.  If ``choices`` is defined, all occurrences of the
            field in the input must have one of the given values (after
            applying ``type``) or else an
            `~headerparser.errors.InvalidChoiceError` is raised.

        :param bool multiple: If `True`, the header field will be allowed to
            occur more than once in the input, and all of the field's values
            will be stored in a list.  If `False` (the default), a
            `~headerparser.errors.DuplicateFieldError` will be raised if the
            field occurs more than once in the input.

        :return: `None`
        :raises ValueError:
            - if another field with the same name or ``dest`` was already
              defined
            - if ``dest`` is not one of the field's names and `add_additional`
              is enabled
            - if ``default`` is defined and ``required`` is true
            - if ``choices`` is an empty sequence
        :raises TypeError: if ``name`` or one of the ``altnames`` is not a
            string
        """

        kwargs.setdefault('dest', name)
        hd = NamedField(name=name, **kwargs)
        normed = set(map(self.normalizer, (name,) + altnames))
        # Error before modifying anything:
        redefs = [n for n in self.fielddefs if n in normed]
        if redefs:
            raise ValueError('field defined more than once: ' + repr(redefs[0]))
        if self.normalizer(hd.dest) in self.dests:
            raise ValueError('destination defined more than once: '
                             + repr(hd.dest))
        if self.normalizer(hd.dest) not in normed:
            if self.additional is not None:
                raise ValueError('add_additional and `dest` are mutually exclusive')
            self.custom_dests = True
        for n in normed:
            self.fielddefs[n] = hd
        self.dests.add(self.normalizer(hd.dest))

    def add_additional(self, allow=True, **kwargs):
        if allow:
            if self.custom_dests:
                raise ValueError('add_additional and `dest` are mutually exclusive')
            self.additional = FieldDef(**kwargs)
{'fraction_non_alphanumeric': 0.0761396099863371, 'fraction_numerical': 0.0009936653831822133, 'mean_word_length': 2.3438538205980066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166787', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2050, 'n_words': 905}