<commit_before>import re
import logging

from share.normalize import ctx, tools
from share.normalize.parsers import Parser
from share.normalize.utils import format_doi_as_url


logger = logging.getLogger(__name__)

THE_REGEX = re.compile(r'(^the\s|\sthe\s)')
DATE_REGEX = re.compile(r'(^[^a-zA-Z]+$)')


def force_text(data):
    if isinstance(data, str):
        return data
    if isinstance(data, dict):
        if '#text' in data:
            return data['#text']
        raise Exception('#text is not in {}'.format(data))
    if isinstance(data, list):
        for item in data:
            try:
                return force_text(item)
            except Exception:
                pass
        raise Exception('No value in list {} is a string.'.format(data))
    raise Exception('{} is not a string or a dictionary.'.format(data))


class Link(Parser):
    url = tools.RunPython(
        'format_link',
        tools.RunPython(
            force_text,
            ctx
        )
    )
    type = tools.Static('doi')

    def format_link(self, link):
        return format_doi_as_url(self, link)


class AlternateLink(Parser):
    schema = 'Link'

    url = tools.RunPython(
        force_text,
        ctx
    )
    type = tools.Static('misc')


class RelatedLink(Parser):
    schema = 'Link'

    url = tools.RunPython(
        force_text,
        ctx
    )
    type = tools.RunPython('lower', tools.Try(ctx['@relatedIdentifierType']))

    def lower(self, type):
        return type.lower()


class ThroughLinks(Parser):
    link = tools.Delegate(Link, ctx)


class ThroughAlternateLinks(Parser):
    schema = 'ThroughLinks'

    link = tools.Delegate(AlternateLink, ctx)


class ThroughRelatedLinks(Parser):
    schema = 'ThroughLinks'

    link = tools.Delegate(RelatedLink, ctx)


class Affiliation(Parser):
    pass


class ContributorInstitution(Parser):
    schema = 'Institution'

    name = ctx.contributorName

    class Extra:
        contributor_type = tools.Try(ctx.contributorType)


class ContributorOrganization(Parser):
    schema = 'Organization'

    name = ctx.contributorName

    class Extra:
        contributor_type = tools.Try(ctx.contributorType)


class CreatorInstitution(Parser):
    schema = 'Institution'

    name = ctx.creatorName


class CreatorOrganization(Parser):
    schema = 'Organization'

    name = ctx


class Identifier(Parser):
    base_url = tools.Try(ctx['@schemeURI'])
    url = tools.Join(tools.Concat(tools.Try(ctx['@schemeURI']), tools.Try(ctx['#text'])), joiner='/')


class ThroughIdentifiers(Parser):
    identifier = tools.Delegate(Identifier, ctx)


class ContributorPerson(Parser):
    schema = 'Person'

    suffix = tools.ParseName(ctx.contributorName).suffix
    family_name = tools.ParseName(ctx.contributorName).last
    given_name = tools.ParseName(ctx.contributorName).first
    additional_name = tools.ParseName(ctx.contributorName).middle
    identifiers = tools.Map(tools.Delegate(ThroughIdentifiers), tools.Concat(tools.Try(ctx, 'nameIdentifier')))

    class Extra:
        name_identifier = tools.Try(
            tools.RunPython(
                force_text,
                ctx.nameIdentifier
            )
        )
        name_identifier_scheme = tools.Try(ctx.nameIdentifier['@nameIdentifierScheme'])
        name_identifier_scheme_uri = tools.Try(ctx.nameIdentifier['@schemeURI'])
        contributor_type = tools.Try(ctx.contributorType)


class CreatorPerson(Parser):
    schema = 'Person'

    suffix = tools.ParseName(ctx.creatorName).suffix
    family_name = tools.ParseName(ctx.creatorName).last
    given_name = tools.ParseName(ctx.creatorName).first
    additional_name = tools.ParseName(ctx.creatorName).middle
    affiliations = tools.Map(tools.Delegate(Affiliation.using(entity=tools.Delegate(CreatorOrganization))), tools.Concat(tools.Try(
        tools.RunPython(
            force_text,
            ctx.affiliation
        )
    )))
    identifiers = tools.Map(tools.Delegate(ThroughIdentifiers), tools.Concat(tools.Try(ctx, 'nameIdentifier')))

    class Extra:
        name_identifier = tools.Try(<commit_msg>datacite clean up<commit_after>import re
import logging

from share.normalize import ctx, tools
from share.normalize.parsers import Parser
from share.normalize.utils import format_doi_as_url


logger = logging.getLogger(__name__)

THE_REGEX = re.compile(r'(^the\s|\sthe\s)')
DATE_REGEX = re.compile(r'(^[^a-zA-Z]+$)')


def force_text(data):
    if isinstance(data, str):
        return data
    if isinstance(data, dict):
        if '#text' in data:
            return data['#text']
        raise Exception('#text is not in {}'.format(data))
    if isinstance(data, list):
        for item in data:
            try:
                return force_text(item)
            except Exception:
                pass
        raise Exception('No value in list {} is a string.'.format(data))
    raise Exception('{} is not a string or a dictionary.'.format(data))


class Link(Parser):
    url = tools.RunPython(
        'format_link',
        tools.RunPython(
            force_text,
            ctx
        )
    )
    type = tools.Static('doi')

    def format_link(self, link):
        return format_doi_as_url(self, link)


class AlternateLink(Parser):
    schema = 'Link'

    url = tools.RunPython(
        force_text,
        ctx
    )
    type = tools.Static('misc')


class RelatedLink(Parser):
    schema = 'Link'

    url = tools.RunPython(
        force_text,
        ctx
    )
    type = tools.RunPython('lower', tools.Try(ctx['@relatedIdentifierType']))

    def lower(self, type):
        return type.lower()


class ThroughLinks(Parser):
    link = tools.Delegate(Link, ctx)


class ThroughAlternateLinks(Parser):
    schema = 'ThroughLinks'

    link = tools.Delegate(AlternateLink, ctx)


class ThroughRelatedLinks(Parser):
    schema = 'ThroughLinks'

    link = tools.Delegate(RelatedLink, ctx)


class Affiliation(Parser):
    pass


class ContributorInstitution(Parser):
    schema = 'Institution'

    name = ctx.contributorName

    class Extra:
        contributor_type = tools.Try(ctx.contributorType)


class ContributorOrganization(Parser):
    schema = 'Organization'

    name = ctx.contributorName

    class Extra:
        contributor_type = tools.Try(ctx.contributorType)


class CreatorInstitution(Parser):
    schema = 'Institution'

    name = ctx.creatorName


class CreatorOrganization(Parser):
    schema = 'Organization'

    name = ctx


class Identifier(Parser):
    base_url = tools.Try(ctx['@schemeURI'])
    url = tools.Join(tools.Concat(tools.Try(ctx['@schemeURI']), tools.Try(ctx['#text'])), joiner='/')


class ThroughIdentifiers(Parser):
    identifier = tools.Delegate(Identifier, ctx)


class ContributorPerson(Parser):
    schema = 'Person'

    suffix = tools.ParseName(ctx.contributorName).suffix
    family_name = tools.ParseName(ctx.contributorName).last
    given_name = tools.ParseName(ctx.contributorName).first
    additional_name = tools.ParseName(ctx.contributorName).middle
    identifiers = tools.Map(tools.Delegate(ThroughIdentifiers), tools.Try(ctx.nameIdentifier))

    class Extra:
        name_identifier = tools.Try(
            tools.RunPython(
                force_text,
                ctx.nameIdentifier
            )
        )
        name_identifier_scheme = tools.Try(ctx.nameIdentifier['@nameIdentifierScheme'])
        name_identifier_scheme_uri = tools.Try(ctx.nameIdentifier['@schemeURI'])
        contributor_type = tools.Try(ctx.contributorType)


class CreatorPerson(Parser):
    schema = 'Person'

    suffix = tools.ParseName(ctx.creatorName).suffix
    family_name = tools.ParseName(ctx.creatorName).last
    given_name = tools.ParseName(ctx.creatorName).first
    additional_name = tools.ParseName(ctx.creatorName).middle
    affiliations = tools.Map(tools.Delegate(Affiliation.using(entity=tools.Delegate(CreatorOrganization))), tools.Concat(tools.Try(
        tools.RunPython(
            force_text,
            ctx.affiliation
        )
    )))
    identifiers = tools.Map(tools.Delegate(ThroughIdentifiers), tools.Try(ctx.nameIdentifier))

    class Extra:
        name_identifier = tools.Try(
{'fraction_non_alphanumeric': 0.1116813294232649, 'fraction_numerical': 0.0, 'mean_word_length': 3.0419753086419754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2998737', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2371, 'n_words': 477}