<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

# from wtforms.validators import ValidationError, StopValidation, Regexp
from werkzeug import MultiDict
from invenio.utils.datacite import DataciteMetadata
from invenio.utils.sherpa_romeo import SherpaRomeoSearch
from invenio.modules.records.api import get_record
from invenio.utils import persistentid as pidutils

#
# General purpose processors
#


def replace_field_data(field_name, getter=None):
    """Return a processor.

    This will replace the given field names value with the value from the field
    where the processor is installed.
    """
    def _inner(form, field, submit=False, fields=None):
        getattr(form, field_name).data = getter(field) if getter else \
            field.data
    return _inner


def set_flag(flag_name):
    """ Return processor which will set a given flag on a field."""
    def _inner(form, field, submit=False, fields=None):
        setattr(field.flags, flag_name, True)
    return _inner


#
# PID processors
#
class PidSchemeDetection(object):

    """Detect persistent identifier scheme and store it in another field."""

    def __init__(self, set_field=None):
        self.set_field = set_field

    def __call__(self, form, field, submit=False, fields=None):
        if field.data:
            schemes = pidutils.detect_identifier_schemes(field.data)
            if schemes:
                getattr(form, self.set_field).data = schemes[0]
            else:
                getattr(form, self.set_field).data = ''


class PidNormalize(object):

    """Normalize a persistent identifier."""

    def __init__(self, scheme_field=None):
        self.scheme_field = scheme_field

    def __call__(self, form, field, submit=False, fields=None):
        scheme = getattr(form, self.scheme_field).data
        if scheme:
            field.data = pidutils.normalize_pid(field.data, scheme=scheme)


#
# DOI-related processors
#
<commit_msg>deposit: pid processor normalization enhancement<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

# from wtforms.validators import ValidationError, StopValidation, Regexp
from werkzeug import MultiDict
from invenio.utils.datacite import DataciteMetadata
from invenio.utils.sherpa_romeo import SherpaRomeoSearch
from invenio.modules.records.api import get_record
from invenio.utils import persistentid as pidutils

#
# General purpose processors
#


def replace_field_data(field_name, getter=None):
    """Return a processor.

    This will replace the given field names value with the value from the field
    where the processor is installed.
    """
    def _inner(form, field, submit=False, fields=None):
        getattr(form, field_name).data = getter(field) if getter else \
            field.data
    return _inner


def set_flag(flag_name):
    """Return processor which will set a given flag on a field."""
    def _inner(form, field, submit=False, fields=None):
        setattr(field.flags, flag_name, True)
    return _inner


#
# PID processors
#
class PidSchemeDetection(object):

    """Detect persistent identifier scheme and store it in another field."""

    def __init__(self, set_field=None):
        self.set_field = set_field

    def __call__(self, form, field, submit=False, fields=None):
        if field.data:
            schemes = pidutils.detect_identifier_schemes(field.data)
            if schemes:
                getattr(form, self.set_field).data = schemes[0]
            else:
                getattr(form, self.set_field).data = ''


class PidNormalize(object):

    """Normalize a persistent identifier."""

    def __init__(self, scheme_field=None, scheme=None):
        self.scheme_field = scheme_field
        self.scheme = scheme

    def __call__(self, form, field, submit=False, fields=None):
        scheme = None
        if self.scheme_field:
            scheme = getattr(form, self.scheme_field).data
        elif self.scheme:
            scheme = self.scheme
        else:
            schemes = pidutils.detect_identifier_schemes(field.data)
            if schemes:
                scheme = schemes[0]
        if scheme:
            if field.data:
                field.data = pidutils.normalize_pid(field.data, scheme=scheme)


#
# DOI-related processors
#

{'fraction_non_alphanumeric': 0.08751519361000173, 'fraction_numerical': 0.008855704115297795, 'mean_word_length': 3.2921013412816693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1157143', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1586, 'n_words': 586}