<commit_before>from datetime import datetime
import logging

from biisan.models import (
    Comment, Paragraph, Section, BulletList, ListItem, Target, Raw, Image, BlockQuote, Title,
    LiteralBlock, Figure, Caption, Table, Thead, Tbody, Tgroup, ColSpec, Row, Entry, EnumeratedList, Transition,
    Topic, SubstitutionDefinition, Note, DefinitionList, DefinitionListItem, Term, Definition,
    Strong, Emphasis, Reference
)

logger = logging.getLogger(__name__)


def _debug(elm):
    logger.debug('-' * 20)
    logger.debug('Tag: {0}'.format(elm.tag))
    logger.debug('text: {0}'.format(elm.text))
    logger.debug('------- items --------------')
    logger.debug(elm.items())
    logger.debug('------- getchildren --------')
    logger.debug(elm.getchildren())
    logger.debug('-' * 20)


def process_field_name(elm, registry, container):
    return elm.text


def process_field_body(elm, registry, container):
    res = []
    for x in elm.getchildren():
        if 'field_list' == x.tag:
            logger.warn("Ignore field_list in field_body's child")
        else:
            res.append(x.text)
    return res


def process_list_item(elm, registry, container):
    list_item = ListItem()
    container.add_content(list_item)
    for _elm in elm.getchildren():
        registry.process(_elm, list_item)


def process_bullet_list(elm, registry, container):
    bullet_list = BulletList()
    container.add_content(bullet_list)
    for _elm in elm.getchildren():
        registry.process(_elm, bullet_list)


def process_definition_list(elm, registry, container):
    definition_list = DefinitionList()
    container.add_content(definition_list)
    for _elm in elm.getchildren():
        registry.process(_elm, definition_list)


def process_definition(elm, registry, container):
    definition = Definition()
    container.definition = definition
    for _elm in elm.getchildren():
        registry.process(_elm, definition)


def process_definition_list_item(elm, registry, container):
    definition_list_item = DefinitionListItem()
    container.add_content(definition_list_item)
    for _elm in elm.getchildren():
        if _elm.tag == 'term':
            term = Term()
            term.text = _elm.text
            definition_list_item.term = term
        elif _elm.tag == 'definition':
            registry.process(_elm, definition_list_item)


def process_target(elm, registry, container):
    target = Target()
    container.add_content(target)
    for subitem in elm.items():
        if subitem[0] == 'ids':
            target.ids = subitem[1]
        elif subitem[0] == 'names':
            target.names = subitem[1]
        elif subitem[0] == 'uri' or subitem[0] == 'refuri':
            target.uri = subitem[1]


def process_reference(elm, registry, container):
    reference = Reference()
    reference.text = elm.text
    container.add_content(reference)
    for subitem in elm.items():
        if subitem[0] == 'name':
            reference.name = subitem[1]
        elif subitem[0] == 'uri' or subitem[0] == 'refuri':
            reference.uri = subitem[1]


def process_raw(elm, registry, container):
    raw = Raw()
    container.add_content(raw)
    for subitem in elm.items():
        if subitem[0] == 'format':
            raw.format = subitem[1]
    raw.text = elm.text


def process_image(elm, registry, container):
    img = Image()
    container.add_content(img)
    for subitem in elm.items():
        if subitem[0] == 'alt':
            img.alt = subitem[1]
        elif subitem[0] == 'witdh':
            img.witdh = subitem[1]
        elif subitem[1] == 'height':
            img.height = subitem[1]
        elif subitem[0] == 'uri':
            img.uri = subitem[1]


def process_block_quote(elm, registry, container):
    block_quote = BlockQuote()
    container.add_content(block_quote)
    for _elm in elm.getchildren():
        registry.process(_elm, block_quote)


def process_literal_block(elm, registry, container):
    literal_block = LiteralBlock()
    container.add_content(literal_block)
    literal_block.text = elm.text<commit_msg>key is first item<commit_after>from datetime import datetime
import logging

from biisan.models import (
    Comment, Paragraph, Section, BulletList, ListItem, Target, Raw, Image, BlockQuote, Title,
    LiteralBlock, Figure, Caption, Table, Thead, Tbody, Tgroup, ColSpec, Row, Entry, EnumeratedList, Transition,
    Topic, SubstitutionDefinition, Note, DefinitionList, DefinitionListItem, Term, Definition,
    Strong, Emphasis, Reference
)

logger = logging.getLogger(__name__)


def _debug(elm):
    logger.debug('-' * 20)
    logger.debug('Tag: {0}'.format(elm.tag))
    logger.debug('text: {0}'.format(elm.text))
    logger.debug('------- items --------------')
    logger.debug(elm.items())
    logger.debug('------- getchildren --------')
    logger.debug(elm.getchildren())
    logger.debug('-' * 20)


def process_field_name(elm, registry, container):
    return elm.text


def process_field_body(elm, registry, container):
    res = []
    for x in elm.getchildren():
        if 'field_list' == x.tag:
            logger.warn("Ignore field_list in field_body's child")
        else:
            res.append(x.text)
    return res


def process_list_item(elm, registry, container):
    list_item = ListItem()
    container.add_content(list_item)
    for _elm in elm.getchildren():
        registry.process(_elm, list_item)


def process_bullet_list(elm, registry, container):
    bullet_list = BulletList()
    container.add_content(bullet_list)
    for _elm in elm.getchildren():
        registry.process(_elm, bullet_list)


def process_definition_list(elm, registry, container):
    definition_list = DefinitionList()
    container.add_content(definition_list)
    for _elm in elm.getchildren():
        registry.process(_elm, definition_list)


def process_definition(elm, registry, container):
    definition = Definition()
    container.definition = definition
    for _elm in elm.getchildren():
        registry.process(_elm, definition)


def process_definition_list_item(elm, registry, container):
    definition_list_item = DefinitionListItem()
    container.add_content(definition_list_item)
    for _elm in elm.getchildren():
        if _elm.tag == 'term':
            term = Term()
            term.text = _elm.text
            definition_list_item.term = term
        elif _elm.tag == 'definition':
            registry.process(_elm, definition_list_item)


def process_target(elm, registry, container):
    target = Target()
    container.add_content(target)
    for subitem in elm.items():
        if subitem[0] == 'ids':
            target.ids = subitem[1]
        elif subitem[0] == 'names':
            target.names = subitem[1]
        elif subitem[0] == 'uri' or subitem[0] == 'refuri':
            target.uri = subitem[1]


def process_reference(elm, registry, container):
    reference = Reference()
    reference.text = elm.text
    container.add_content(reference)
    for subitem in elm.items():
        if subitem[0] == 'name':
            reference.name = subitem[1]
        elif subitem[0] == 'uri' or subitem[0] == 'refuri':
            reference.uri = subitem[1]


def process_raw(elm, registry, container):
    raw = Raw()
    container.add_content(raw)
    for subitem in elm.items():
        if subitem[0] == 'format':
            raw.format = subitem[1]
    raw.text = elm.text


def process_image(elm, registry, container):
    img = Image()
    container.add_content(img)
    for subitem in elm.items():
        if subitem[0] == 'alt':
            img.alt = subitem[1]
        elif subitem[0] == 'witdh':
            img.witdh = subitem[1]
        elif subitem[0] == 'height':
            img.height = subitem[1]
        elif subitem[0] == 'uri':
            img.uri = subitem[1]


def process_block_quote(elm, registry, container):
    block_quote = BlockQuote()
    container.add_content(block_quote)
    for _elm in elm.getchildren():
        registry.process(_elm, block_quote)


def process_literal_block(elm, registry, container):
    literal_block = LiteralBlock()
    container.add_content(literal_block)
    literal_block.text = elm.text
{'fraction_non_alphanumeric': 0.12475538160469667, 'fraction_numerical': 0.00684931506849315, 'mean_word_length': 2.9732750242954324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3686143', 'n_tokens_mistral': 2514, 'n_tokens_neox': 2451, 'n_words': 582}