<commit_before>import json
from lxml import etree


def cdata(text):
    """
    Creates a CDATA string that is capable of containing other CDATA blocks that
    are escaped automatically.

    Can handle bytes, strings and Nones gracefully, always resulting in a
    CDATA'd string.

    :param text: The text to CDATA-rize.
    :return: The text wrapped in CDATA tags, with nested CDATA being escaped.
    """
    # import code
    # code.interact(local=locals())
    if isinstance(text, bytes):
        text = text.decode('utf-8')
    elif text is None:
        text = ''
    escaped = text.replace(']]>', ']]&gt;')
    return etree.CDATA(escaped)


def add_unused_item_children(item):
    """
    Adds required, but unused elements to the item element.

    :param item: The XML element corresponding to the item being exported.
    """
    etree.SubElement(item, 'allImages')
    etree.SubElement(item, 'allAttributes')
    etree.SubElement(item, 'allKeywords')
    etree.SubElement(item, 'salesFrequencies')
    etree.SubElement(item, 'usergroups')
    add_single_nested_data(item, 'prices', 'price', str(0.0))


def add_properties(item, properties):
    """
    Adds the properties provided as a flat dictionary to the item. Properties
    with an empty value are not added for schema conformance.

    :param item: The item to add properties to.
    :param properties: The properties to add to the item.
    """
    all_properties = etree.SubElement(item, 'allProperties')
    props = etree.SubElement(all_properties, 'properties')

    for key, value in properties.items():
        if value is None or len(value) < 1:
            continue

        prop = etree.SubElement(props, 'property')
        prop_key = etree.SubElement(prop, 'key')
        prop_key.text = cdata(key)
        prop_value = etree.SubElement(prop, 'value')
        prop_value.text = cdata(value)


def add_regular_item_values(item, page):
    """
    Adds simple, regular values to the item, including the path of the page,
    title, summary, full text, and update date.

    :param item: The item to modify.
    :param page: The page sourcing the data.
    """
    all_ordernumbers = etree.SubElement(item, 'allOrdernumbers')
    add_single_nested_data(all_ordernumbers, 'ordernumbers', 'ordernumber',
                           page.path)

    add_single_nested_data(item, 'names', 'name', page.title)
    add_single_nested_data(item, 'summaries', 'summary', page.description)
    add_single_nested_data(item, 'descriptions', 'description', page.text)

    date_addeds = etree.SubElement(item, 'dateAddeds')
    if page.created_at is not None:
        # Use the update date instead of the creation date, so search results<commit_msg>Only export dateAdded if it is set<commit_after>import json
from lxml import etree


def cdata(text):
    """
    Creates a CDATA string that is capable of containing other CDATA blocks that
    are escaped automatically.

    Can handle bytes, strings and Nones gracefully, always resulting in a
    CDATA'd string.

    :param text: The text to CDATA-rize.
    :return: The text wrapped in CDATA tags, with nested CDATA being escaped.
    """
    # import code
    # code.interact(local=locals())
    if isinstance(text, bytes):
        text = text.decode('utf-8')
    elif text is None:
        text = ''
    escaped = text.replace(']]>', ']]&gt;')
    return etree.CDATA(escaped)


def add_unused_item_children(item):
    """
    Adds required, but unused elements to the item element.

    :param item: The XML element corresponding to the item being exported.
    """
    etree.SubElement(item, 'allImages')
    etree.SubElement(item, 'allAttributes')
    etree.SubElement(item, 'allKeywords')
    etree.SubElement(item, 'salesFrequencies')
    etree.SubElement(item, 'usergroups')
    add_single_nested_data(item, 'prices', 'price', str(0.0))


def add_properties(item, properties):
    """
    Adds the properties provided as a flat dictionary to the item. Properties
    with an empty value are not added for schema conformance.

    :param item: The item to add properties to.
    :param properties: The properties to add to the item.
    """
    all_properties = etree.SubElement(item, 'allProperties')
    props = etree.SubElement(all_properties, 'properties')

    for key, value in properties.items():
        if value is None or len(value) < 1:
            continue

        prop = etree.SubElement(props, 'property')
        prop_key = etree.SubElement(prop, 'key')
        prop_key.text = cdata(key)
        prop_value = etree.SubElement(prop, 'value')
        prop_value.text = cdata(value)


def add_regular_item_values(item, page):
    """
    Adds simple, regular values to the item, including the path of the page,
    title, summary, full text, and update date.

    :param item: The item to modify.
    :param page: The page sourcing the data.
    """
    all_ordernumbers = etree.SubElement(item, 'allOrdernumbers')
    add_single_nested_data(all_ordernumbers, 'ordernumbers', 'ordernumber',
                           page.path)

    add_single_nested_data(item, 'names', 'name', page.title)
    add_single_nested_data(item, 'summaries', 'summary', page.description)
    add_single_nested_data(item, 'descriptions', 'description', page.text)

    date_addeds = etree.SubElement(item, 'dateAddeds')
    if page.updated_at is not None:
        # Use the update date instead of the creation date, so search results
{'fraction_non_alphanumeric': 0.10049514028974876, 'fraction_numerical': 0.0014670823399963323, 'mean_word_length': 3.2776470588235296, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3277555', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1599, 'n_words': 521}