<commit_before>from share.normalize import links
from share.normalize import parsers
from share.normalize.normalizer import Normalizer


# NOTE: Context is a thread local singleton
# It is asigned to ctx here just to keep a family interface
ctx = links.Context()


class OAIPerson(parsers.Parser):
    schema = 'Person'

    suffix = links.ParseName(ctx).suffix
    family_name = links.ParseName(ctx).last
    given_name = links.ParseName(ctx).first
    additional_name = links.ParseName(ctx).middle


class OAIContributor(parsers.Parser):
    schema = 'Contributor'
    Person = OAIPerson

    person = ctx
    cited_name = ctx
    order_cited = ctx('index')


class OAIPublisher(parsers.Parser):
    schema = 'Publisher'
    name = ctx


class OAIAssociation(parsers.Parser):
    schema = 'Association'
    # entity = links.Generic('entity')(ctx)


class OAITaxonomy(parsers.Parser):
    schema = 'Taxonomy'
    name = ctx


class OAITag(parsers.Parser):
    schema = 'Tag'
    name = ctx


class OAIThroughTags(parsers.Parser):
    schema = 'ThroughTags'
    tag = OAITag(ctx)


class OAICreativeWork(parsers.Parser):
    schema = 'CreativeWork'

    title = ctx.record.metadata['oai_dc:dc']['dc:title']
    rights = ctx.record.metadata['oai_dc:dc']['dc:rights']
    language = ctx.record.metadata['oai_dc:dc']['dc:language']
    description = links.Join(links.Maybe(ctx.record.metadata['oai_dc:dc'], 'dc:description'))

    published = links.ParseDate(ctx.record.metadata['oai_dc:dc']['dc:date'][0])<commit_msg>Update OAI normalizer<commit_after>from share.normalize import links
from share.normalize import parsers
from share.normalize.normalizer import Normalizer


# NOTE: Context is a thread local singleton
# It is asigned to ctx here just to keep a family interface
ctx = links.Context()


class OAIPerson(parsers.Parser):
    schema = 'Person'

    suffix = links.ParseName(ctx).suffix
    family_name = links.ParseName(ctx).last
    given_name = links.ParseName(ctx).first
    additional_name = links.ParseName(ctx).middle


class OAIContributor(parsers.Parser):
    schema = 'Contributor'

    person = links.Delegate(OAIPerson, ctx)
    cited_name = ctx
    order_cited = ctx('index')


class OAIPublisher(parsers.Parser):
    schema = 'Publisher'
    name = ctx


class OAIAssociation(parsers.Parser):
    schema = 'Association'


class OAITaxonomy(parsers.Parser):
    schema = 'Taxonomy'
    name = ctx


class OAITag(parsers.Parser):
    schema = 'Tag'
    name = ctx


class OAIThroughTags(parsers.Parser):
    schema = 'ThroughTags'
    tag = links.Delegate(OAITag, ctx)


class OAICreativeWork(parsers.Parser):
    schema = 'CreativeWork'

    title = ctx.record.metadata['oai_dc:dc']['dc:title']
    rights = ctx.record.metadata['oai_dc:dc']['dc:rights']
    language = ctx.record.metadata['oai_dc:dc']['dc:language']
    description = links.Join(links.Maybe(ctx.record.metadata['oai_dc:dc'], 'dc:description'))

    published = links.ParseDate(ctx.record.metadata['oai_dc:dc']['dc:date'][0])
{'fraction_non_alphanumeric': 0.1234240212342402, 'fraction_numerical': 0.0006635700066357001, 'mean_word_length': 4.243478260869566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083449', 'n_tokens_mistral': 1015, 'n_tokens_neox': 954, 'n_words': 203}