<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import dhtmlparser


# Functions & objects =========================================================
def _remove_hairs(inp, hairs="/:;,- []<>()"):
    """
    Remove "special" characters from beginning and the end of the `inp`. For
    example ``,a-sd,-/`` -> ``a-sd``.

    Args:
        inp (str): Input string.

    Returns:
        str: Cleaned string.
    """
    while inp and inp[-1] in hairs:
        inp = inp[:-1]

    while inp and inp[0] in hairs:
        inp = inp[1:]

    return inp


def insert_tag(tag, before, root):
    """
    Insert `tag` before `before` tag if present. If not, insert it into `root`.

    Args:
        tag (obj): HTMLElement instance.
        before (obj): HTMLElement instance.
        root (obj): HTMLElement instance.
    """
    if not before:
        root.childs.append(tag)
        tag.parent = root
        return

    before = before[0] if type(before) in [tuple, list] else before

    # put it before first existing identifier
    parent = before.parent
    parent.childs.insert(
        parent.childs.index(before),
        tag
    )
    tag.parent = parent


def transform_content(tags, content_transformer):
    """
    Transform content in all `tags` using result of `content_transformer(tag)`
    call.

    Args:
        tags (obj/list): HTMLElement instance, or list of HTMLElement
                         instances.
        content_transformer (function): Function which is called as
                                        ``content_transformer(tag)``.
    """
    if type(tags) not in [tuple, list]:
        tags = [tags]

    for tag in tags:
        tag.childs = [
            dhtmlparser.HTMLElement(content_transformer(tag))
        ]


def postprocess_mods_volume(mods, uuid):
    """
    Fix bugs in `mods` produced by XSLT template.

    Args:
        mods (str): XML string generated by XSLT template.
        uuid (str): UUID of the package.

    Returns:
        str: Updated XML.
    """
    # do not parse already parsed dom's
    dom = mods
    if not isinstance(mods, dhtmlparser.HTMLElement):
        dom = dhtmlparser.parseString(mods)

    dhtmlparser.makeDoubleLinked(dom)

    # add missing parameter
    mods_tag = dom.find("mods:mods")
    if mods_tag:
        params = mods_tag[0].params

        # add missing attributes
        params["ID"] = "MODS_VOLUME_0001"
        params["xmlns:mods"] = "http://www.loc.gov/mods/v3"
        params["xmlns:xlink"] = "http://www.w3.org/1999/xlink"
        params["xmlns:xsi"] = "http://www.w3.org/2001/XMLSchema-instance"
        params["xsi:schemaLocation"] = (
            "http://www.w3.org/2001/XMLSchema-instance "
            "http://www.w3.org/2001/XMLSchema.xsd "
            "http://www.loc.gov/mods/v3 "
            "http://www.loc.gov/standards/mods/v3/mods-3-4.xsd "
            "http://www.w3.org/1999/xlink http://www.w3.org/1999/xlink.xsd"
        )

    # fix invalid type= paramater
    placeterm_tag = dom.match(
        "mods:originInfo",
        "mods:place",
        ["mods:placeTerm", {"authority": "marccountry"}]
    )
    if placeterm_tag:
        placeterm_tag[0].params["type"] = "code"

    # add identifier to the section with identifiers
    if uuid:
        uuid_tag = dhtmlparser.HTMLElement(
            "mods:identifier",
            {"type": "uuid"},
            [dhtmlparser.HTMLElement(uuid)]
        )
        insert_tag(uuid_tag, dom.find("mods:identifier"), mods_tag)

    # add marccountry if not found
    marccountry = dom.find("mods:placeTerm", {"authority": "marccountry"})
    if not marccountry:
        marccountry_tag = dhtmlparser.HTMLElement(
            "mods:place",
            [
                dhtmlparser.HTMLElement(
                    "mods:placeterm",
                    {"type": "code", "authority": "marccountry"},
                    [dhtmlparser.HTMLElement("xr")]
                )
            ]
        )
        insert_tag(
            marccountry_tag,
            dom.match("mods:mods", "mods:originInfo", "mods:place"),
            dom.find("mods:originInfo")[0]
        )
<commit_msg>#19: Fixed placeTerm tag.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import dhtmlparser


# Functions & objects =========================================================
def _remove_hairs(inp, hairs="/:;,- []<>()"):
    """
    Remove "special" characters from beginning and the end of the `inp`. For
    example ``,a-sd,-/`` -> ``a-sd``.

    Args:
        inp (str): Input string.

    Returns:
        str: Cleaned string.
    """
    while inp and inp[-1] in hairs:
        inp = inp[:-1]

    while inp and inp[0] in hairs:
        inp = inp[1:]

    return inp


def insert_tag(tag, before, root):
    """
    Insert `tag` before `before` tag if present. If not, insert it into `root`.

    Args:
        tag (obj): HTMLElement instance.
        before (obj): HTMLElement instance.
        root (obj): HTMLElement instance.
    """
    if not before:
        root.childs.append(tag)
        tag.parent = root
        return

    before = before[0] if type(before) in [tuple, list] else before

    # put it before first existing identifier
    parent = before.parent
    parent.childs.insert(
        parent.childs.index(before),
        tag
    )
    tag.parent = parent


def transform_content(tags, content_transformer):
    """
    Transform content in all `tags` using result of `content_transformer(tag)`
    call.

    Args:
        tags (obj/list): HTMLElement instance, or list of HTMLElement
                         instances.
        content_transformer (function): Function which is called as
                                        ``content_transformer(tag)``.
    """
    if type(tags) not in [tuple, list]:
        tags = [tags]

    for tag in tags:
        tag.childs = [
            dhtmlparser.HTMLElement(content_transformer(tag))
        ]


def postprocess_mods_volume(mods, uuid):
    """
    Fix bugs in `mods` produced by XSLT template.

    Args:
        mods (str): XML string generated by XSLT template.
        uuid (str): UUID of the package.

    Returns:
        str: Updated XML.
    """
    # do not parse already parsed dom's
    dom = mods
    if not isinstance(mods, dhtmlparser.HTMLElement):
        dom = dhtmlparser.parseString(mods)

    dhtmlparser.makeDoubleLinked(dom)

    # add missing parameter
    mods_tag = dom.find("mods:mods")
    if mods_tag:
        params = mods_tag[0].params

        # add missing attributes
        params["ID"] = "MODS_VOLUME_0001"
        params["xmlns:mods"] = "http://www.loc.gov/mods/v3"
        params["xmlns:xlink"] = "http://www.w3.org/1999/xlink"
        params["xmlns:xsi"] = "http://www.w3.org/2001/XMLSchema-instance"
        params["xsi:schemaLocation"] = (
            "http://www.w3.org/2001/XMLSchema-instance "
            "http://www.w3.org/2001/XMLSchema.xsd "
            "http://www.loc.gov/mods/v3 "
            "http://www.loc.gov/standards/mods/v3/mods-3-4.xsd "
            "http://www.w3.org/1999/xlink http://www.w3.org/1999/xlink.xsd"
        )

    # fix invalid type= paramater
    placeterm_tag = dom.match(
        "mods:originInfo",
        "mods:place",
        ["mods:placeTerm", {"authority": "marccountry"}]
    )
    if placeterm_tag:
        placeterm_tag[0].params["type"] = "code"

    # add identifier to the section with identifiers
    if uuid:
        uuid_tag = dhtmlparser.HTMLElement(
            "mods:identifier",
            {"type": "uuid"},
            [dhtmlparser.HTMLElement(uuid)]
        )
        insert_tag(uuid_tag, dom.find("mods:identifier"), mods_tag)

    # add marccountry if not found
    marccountry = dom.find("mods:placeTerm", {"authority": "marccountry"})
    if not marccountry:
        marccountry_tag = dhtmlparser.HTMLElement(
            "mods:place",
            [
                dhtmlparser.HTMLElement(
                    "mods:placeTerm",
                    {"type": "code", "authority": "marccountry"},
                    [dhtmlparser.HTMLElement("xr")]
                )
            ]
        )
        insert_tag(
            marccountry_tag,
            dom.match("mods:mods", "mods:originInfo", "mods:place"),
            dom.find("mods:originInfo")[0]
        )

{'fraction_non_alphanumeric': 0.15583958724202626, 'fraction_numerical': 0.011960600375234522, 'mean_word_length': 2.1276127612761275, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 18, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4881974', 'n_tokens_mistral': 2760, 'n_tokens_neox': 2671, 'n_words': 595}