<commit_before>from share.normalize import *  # noqa


class Person(Parser):
    given_name = ParseName(ctx.author_name).first
    family_name = ParseName(ctx.author_name).last


class Contributor(Parser):
    order_cited = ctx('index')
    cited_name = ctx.author_name
    person = Delegate(Person, ctx)


class CreativeWork(Parser):
    # doi = ctx.DOI
    title = ctx.title
    description = ctx.description
    contributors = Map(Delegate(Contributor), ctx.authors)
    published = ParseDate(ctx.published_date)

    class Extra:
        modifed = ParseDate(ctx.modifed_date)<commit_msg>Update figshare normalizer<commit_after>import arrow

import dateparser

from share.normalize import *  # noqa


class Person(Parser):
    given_name = ParseName(ctx.author_name).first
    family_name = ParseName(ctx.author_name).last


class Contributor(Parser):
    order_cited = ctx('index')
    cited_name = ctx.author_name
    person = Delegate(Person, ctx)


class Link(Parser):
    url = ctx
    type = RunPython('get_link_type', ctx)

    def get_link_type(self, link):
        if 'dx.doi.org' in link:
            return 'doi'
        if 'figshare.com' in link:
            return 'provider'
        return 'misc'


class ThroughLinks(Parser):
    link = Delegate(Link, ctx)


class CreativeWork(Parser):
    title = ctx.title
    description = ctx.description
    contributors = Map(Delegate(Contributor), ctx.authors)
    published = RunPython('parse_date', ctx.published_date)
    links = Concat(
        Delegate(ThroughLinks, ctx.url),
        Delegate(ThroughLinks, ctx.DOI),
        Map(Delegate(ThroughLinks), ctx.links)
    )

    class Extra:
        modifed = RunPython('parse_date', ctx.modifed_date)

    def parse_date(self, date_str):
        return arrow.get(dateparser.parse(date_str)).to('UTC').isoformat()
{'fraction_non_alphanumeric': 0.10635964912280702, 'fraction_numerical': 0.0, 'mean_word_length': 3.397590361445783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6798318', 'n_tokens_mistral': 590, 'n_tokens_neox': 559, 'n_words': 119}