<commit_before>#bibframe.reader.util
#FIXME: Rename to bibframe.reader.pipeline (cf Versa) post 1.0
'''
Library of functions that take a prototype link set and generate a transformed link set


'''

import re
import sys
from itertools import product
from enum import Enum #https://docs.python.org/3.4/library/enum.html
from collections import OrderedDict

from versa.pipeline import context as versacontext
from versa import I, VERSA_BASEIRI, ORIGIN, RELATIONSHIP, TARGET, ATTRIBUTES

from bibframe.contrib.datachefids import slugify#, FROM_EMPTY_64BIT_HASH
from bibframe.contrib.datachefids import idgen as default_idgen
from bibframe import BFZ, BL
from bibframe.isbnplus import isbn_list, compute_ean13_check

from bibframe.reader import BOOTSTRAP_PHASE

from amara3 import iri

__all__ = ["bfcontext", "base_transformer", "link", "ignore", "anchor", "target", "origin",
            "all_subfields", "subfield", "values", "relator_property", "replace_from",
            "ifexists", "foreach", "indicator", "materialize", "url", "normalize_isbn",
            "onwork", "oninstance", "lookup", "regex_match_modify", "register_transforms",
            "subfields", "abort_on", "lookup_inline"]

RDA_PARENS_PAT = re.compile('\\(.*\\)')

PYBF_BASE = '"http://bibfra.me/tool/pybibframe/transforms#'
WORK_TYPE = BL + 'Work'
INSTANCE_TYPE = BL + 'Instance'
VTYPE_REL = I(iri.absolutize('type', VERSA_BASEIRI))

#FIXME: Make proper use of subclassing (implementation derivation)
class bfcontext(versacontext):
    def __init__(self, current_link, input_model, output_model, base=None, extras=None, idgen=None, existing_ids=None, logger=None):
        self.current_link = current_link
        self.input_model = input_model
        self.output_model = output_model
        self.base = base
        self.extras = extras or {}
        self.idgen = idgen or default_idgen(base)
        self.existing_ids = existing_ids or set()
        self.logger = logger
        return

    def copy(self, current_link=None, input_model=None, output_model=None, base=None, extras=None, idgen=None, existing_ids=None, logger=None):
        current_link = current_link if current_link else self.current_link
        input_model = input_model if input_model else self.input_model
        output_model = output_model if output_model else self.output_model
        base = base if base else self.base
        extras = extras if extras else self.extras
        idgen = idgen if idgen else self.idgen
        existing_ids = existing_ids if existing_ids else self.existing_ids
        logger = logger if logger else self.logger

        return bfcontext(current_link, input_model, output_model, base=base, extras=extras, idgen=idgen, existing_ids=existing_ids, logger=logger)


#class action(Enum):
#    replace = 1

DEFAULT_REL = object()

#SUBFIELDS_CACHE = {}

#Can't use functools.lru_cache because of the dict (unhashable) arg
#@lru_cache()
def subfields(attrs, code=None, ctx=None):
    result = []
    # If the attributes have their own ordering, use it, otherwise sort
    attrs_in_order = sorted(attrs.items())
    for ix, (k, v) in enumerate(attrs_in_order):
        if '.' in k:
            this_code = k.rsplit('.')[-1]
            if code is None or this_code == code:
                result.append((this_code, v))
                if ctx and 'current-subfield-ix' in ctx.extras:
                    ctx.extras['current-subfield-ix'].append(ix)
    return result


class base_transformer(object):
    def __init__(self, origin_type=None):
        self._origin_type = origin_type
        return

    def __call__(self, origin_type, *args, **kwargs):
        self._origin_type = origin_type
        return

    def derive_origin(self, ctx):
        '''
        Given a pipeline transform context, derive an origin for generated Versa links
        from whether we're meant to deal with work or instance rules
        '''
        #Origins are indexed by resource type
        return ctx.extras['origins'][self._origin_type]
        #workid, iid = ctx.extras[WORK_TYPE], ctx.extras[INSTANCE_TYPE]
        #return {origin_class.work: workid, origin_class.instance: iid}[self._use_origin]

    def link(self, rel=None, value=None, res=False, ignore_refs=True):
        '''
        Create a link based the context's current link, specifying the output link
        IRI and a target value to be constructed for the link

        rel - IRI of the output link
        value - construction of the target value ofthe link. If None, equivalent of
                using target(), i.e. just reuse the target of the context current link
        res = if True mark the link target value as an IRI (i.e. not a plain string)
        '''
        #Delegate the work
        return link(derive_origin=self.derive_origin, rel=rel, value=value, res=res, ignore_refs=ignore_refs)

    def materialize(self, typ, rel=DEFAULT_REL, unique=None, links=None, postprocess=None):
        '''
        Create a new resource related to the origin
        '''
        links = links or {}
        #Delegate the work
        return materialize(typ, rel=rel, derive_origin=self.derive_origin, unique=unique,
                            links=links, postprocess=postprocess)


def link(derive_origin=None, rel=None, value=None, res=False, ignore_refs=True):
    '''
    Create a new link

    :param derive_origin: Versa action function to be invoked in order to
    determine the origin of the main generated link. If none the origin is derived
    from the context given when the materialize action function is called

    :param rel: IRI of the relationship to be created, from the origin,
    or a list of relationship IRIs, each of which will be used to create
    a separate link, or a versa action function to derive this relationship or
    list of relationships at run time, or None. If None, the relationship is derived
    from the context given when the materialize action function is called

    For examples of all these scenarios see marcpatterns.py

    :return: Versa action function to do the actual work
    '''
    def _link(ctx):
        (origin, _, t, a) = ctx.current_link
        if derive_origin:
            #Have enough info to derive the origin from context. Ignore origin in current link
            origin = derive_origin(ctx)

        #If need be call the Versa action function to determine the relationship to the materialized resource
        rels = rel(ctx) if callable(rel) else rel
        if not isinstance(rels, list): rels = [rels]

        values = value(ctx) if callable(value) else (t if value is None else value)
        if not isinstance(values, list): values = [values]

        def recurse_values(vs):
            for v in vs:
                if callable(v):
                    yield from recurse_values(v(ctx))
                else:
                    yield v

        for _value in recurse_values(values):
            if res and not (ignore_refs and not iri.is_absolute(_value)):
                try:
                    _value = I(_value)
                except ValueError:
                    ctx.extras['logger'].warn('Requirement to convert link target to IRI failed for invalid input, causing the corresponding output link to be omitted entirely: {0}'.format(repr((I(origin), I(iri.absolutize(rel, ctx.base)), _value))))
                    #XXX How do we really want to handle this error?
                    return []

            for r in rels:
                ctx.output_model.add(I(origin), I(iri.absolutize(r, ctx.base)), _value, {})

        return

    return _link


def ignore():
    '''
    Action function generator to do nothing, a no-op

    :return: None
    '''
    #Action function generator to do nothing
    def _ignore(ctx):
        #(o, r, t, a) = ctx.current_link
        ctx.extras['logger'].debug(' {0} ignored by rule'.format(ctx.extras['match-spec']))
        return
    return _ignore


from bibframe import *
WORK_TYPE = BL+'Work'

def anchor(rtype=WORK_TYPE):<commit_msg>Softer fail when value IRI conversion fails<commit_after>#bibframe.reader.util
#FIXME: Rename to bibframe.reader.pipeline (cf Versa) post 1.0
'''
Library of functions that take a prototype link set and generate a transformed link set


'''

import re
import sys
from itertools import product
from enum import Enum #https://docs.python.org/3.4/library/enum.html
from collections import OrderedDict

from versa.pipeline import context as versacontext
from versa import I, VERSA_BASEIRI, ORIGIN, RELATIONSHIP, TARGET, ATTRIBUTES

from bibframe.contrib.datachefids import slugify#, FROM_EMPTY_64BIT_HASH
from bibframe.contrib.datachefids import idgen as default_idgen
from bibframe import BFZ, BL
from bibframe.isbnplus import isbn_list, compute_ean13_check

from bibframe.reader import BOOTSTRAP_PHASE

from amara3 import iri

__all__ = ["bfcontext", "base_transformer", "link", "ignore", "anchor", "target", "origin",
            "all_subfields", "subfield", "values", "relator_property", "replace_from",
            "ifexists", "foreach", "indicator", "materialize", "url", "normalize_isbn",
            "onwork", "oninstance", "lookup", "regex_match_modify", "register_transforms",
            "subfields", "abort_on", "lookup_inline"]

RDA_PARENS_PAT = re.compile('\\(.*\\)')

PYBF_BASE = '"http://bibfra.me/tool/pybibframe/transforms#'
WORK_TYPE = BL + 'Work'
INSTANCE_TYPE = BL + 'Instance'
VTYPE_REL = I(iri.absolutize('type', VERSA_BASEIRI))

#FIXME: Make proper use of subclassing (implementation derivation)
class bfcontext(versacontext):
    def __init__(self, current_link, input_model, output_model, base=None, extras=None, idgen=None, existing_ids=None, logger=None):
        self.current_link = current_link
        self.input_model = input_model
        self.output_model = output_model
        self.base = base
        self.extras = extras or {}
        self.idgen = idgen or default_idgen(base)
        self.existing_ids = existing_ids or set()
        self.logger = logger
        return

    def copy(self, current_link=None, input_model=None, output_model=None, base=None, extras=None, idgen=None, existing_ids=None, logger=None):
        current_link = current_link if current_link else self.current_link
        input_model = input_model if input_model else self.input_model
        output_model = output_model if output_model else self.output_model
        base = base if base else self.base
        extras = extras if extras else self.extras
        idgen = idgen if idgen else self.idgen
        existing_ids = existing_ids if existing_ids else self.existing_ids
        logger = logger if logger else self.logger

        return bfcontext(current_link, input_model, output_model, base=base, extras=extras, idgen=idgen, existing_ids=existing_ids, logger=logger)


#class action(Enum):
#    replace = 1

DEFAULT_REL = object()

#SUBFIELDS_CACHE = {}

#Can't use functools.lru_cache because of the dict (unhashable) arg
#@lru_cache()
def subfields(attrs, code=None, ctx=None):
    result = []
    # If the attributes have their own ordering, use it, otherwise sort
    attrs_in_order = sorted(attrs.items())
    for ix, (k, v) in enumerate(attrs_in_order):
        if '.' in k:
            this_code = k.rsplit('.')[-1]
            if code is None or this_code == code:
                result.append((this_code, v))
                if ctx and 'current-subfield-ix' in ctx.extras:
                    ctx.extras['current-subfield-ix'].append(ix)
    return result


class base_transformer(object):
    def __init__(self, origin_type=None):
        self._origin_type = origin_type
        return

    def __call__(self, origin_type, *args, **kwargs):
        self._origin_type = origin_type
        return

    def derive_origin(self, ctx):
        '''
        Given a pipeline transform context, derive an origin for generated Versa links
        from whether we're meant to deal with work or instance rules
        '''
        #Origins are indexed by resource type
        return ctx.extras['origins'][self._origin_type]
        #workid, iid = ctx.extras[WORK_TYPE], ctx.extras[INSTANCE_TYPE]
        #return {origin_class.work: workid, origin_class.instance: iid}[self._use_origin]

    def link(self, rel=None, value=None, res=False, ignore_refs=True):
        '''
        Create a link based the context's current link, specifying the output link
        IRI and a target value to be constructed for the link

        rel - IRI of the output link
        value - construction of the target value ofthe link. If None, equivalent of
                using target(), i.e. just reuse the target of the context current link
        res = if True mark the link target value as an IRI (i.e. not a plain string)
        '''
        #Delegate the work
        return link(derive_origin=self.derive_origin, rel=rel, value=value, res=res, ignore_refs=ignore_refs)

    def materialize(self, typ, rel=DEFAULT_REL, unique=None, links=None, postprocess=None):
        '''
        Create a new resource related to the origin
        '''
        links = links or {}
        #Delegate the work
        return materialize(typ, rel=rel, derive_origin=self.derive_origin, unique=unique,
                            links=links, postprocess=postprocess)


def link(derive_origin=None, rel=None, value=None, res=False, ignore_refs=True):
    '''
    Create a new link

    :param derive_origin: Versa action function to be invoked in order to
    determine the origin of the main generated link. If none the origin is derived
    from the context given when the materialize action function is called

    :param rel: IRI of the relationship to be created, from the origin,
    or a list of relationship IRIs, each of which will be used to create
    a separate link, or a versa action function to derive this relationship or
    list of relationships at run time, or None. If None, the relationship is derived
    from the context given when the materialize action function is called

    For examples of all these scenarios see marcpatterns.py

    :return: Versa action function to do the actual work
    '''
    def _link(ctx):
        (origin, _, t, a) = ctx.current_link
        if derive_origin:
            #Have enough info to derive the origin from context. Ignore origin in current link
            origin = derive_origin(ctx)

        #If need be call the Versa action function to determine the relationship to the materialized resource
        rels = rel(ctx) if callable(rel) else rel
        if not isinstance(rels, list): rels = [rels]

        values = value(ctx) if callable(value) else (t if value is None else value)
        if not isinstance(values, list): values = [values]

        def recurse_values(vs):
            for v in vs:
                if callable(v):
                    yield from recurse_values(v(ctx))
                else:
                    yield v

        for _value in recurse_values(values):
            if res and not (ignore_refs and not iri.is_absolute(_value)):
                try:
                    _value = I(_value)
                except ValueError:
                    ctx.extras['logger'].warn('Requirement to convert link target to IRI failed for invalid input, causing the corresponding output link to be omitted entirely: {0}'.format(repr((I(origin), I(iri.absolutize(rel, ctx.base)), _value))))
                    #XXX How do we really want to handle this error?
                    #return []
                    continue

            for r in rels:
                ctx.output_model.add(I(origin), I(iri.absolutize(r, ctx.base)), _value, {})

        return

    return _link


def ignore():
    '''
    Action function generator to do nothing, a no-op

    :return: None
    '''
    #Action function generator to do nothing
    def _ignore(ctx):
        #(o, r, t, a) = ctx.current_link
        ctx.extras['logger'].debug(' {0} ignored by rule'.format(ctx.extras['match-spec']))
        return
    return _ignore


from bibframe import *
WORK_TYPE = BL+'Work'

def anchor(rtype=WORK_TYPE):
{'fraction_non_alphanumeric': 0.08804835493519442, 'fraction_numerical': 0.0016201395812562313, 'mean_word_length': 2.9086702386751098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1502835', 'n_tokens_mistral': 4688, 'n_tokens_neox': 4540, 'n_words': 1572}