<commit_before>import html
import logging
import os
import re
from copy import deepcopy
from datetime import datetime
from textwrap import TextWrapper

import dateparser
import lxml
from dateutil.parser import parse as dateutil_parse
from dateutil.tz import gettz
from lxml import etree
from lxml.cssselect import CSSSelector
from lxml.html.clean import Cleaner
from scrapy.loader import ItemLoader
from scrapy.loader.processors import Compose, Identity, Join, MapCompose, TakeFirst
from w3lib.html import remove_tags

from feeds.items import FeedEntryItem, FeedItem
from feeds.settings import get_feeds_settings

logger = logging.getLogger(__name__)


def parse_datetime(date_time, loader_context):
    if isinstance(date_time, datetime):
        return date_time
    elif isinstance(date_time, str):
        try:
            return dateutil_parse(
                date_time.strip(),
                dayfirst=loader_context.get("dayfirst", False),
                yearfirst=loader_context.get("yearfirst", True),
                ignoretz=loader_context.get("ignoretz", False),
            )
        except ValueError:
            # If dateutil can't parse it, it might be a human-readable date.
            return dateparser.parse(date_time)
    else:
        raise ValueError("date_time must be datetime or a str.")


def apply_timezone(date_time, loader_context):
    if not date_time.tzinfo:
        # If date_time object is not aware, apply timezone from loader_context.
        # In case a timezone is not supplied, just assume UTC.
        date_time = date_time.replace(
            tzinfo=gettz(loader_context.get("timezone", "UTC"))
        )
    return date_time


def replace_regex(text, loader_context):
    for pattern, repl in loader_context.get("replace_regex", {}).items():
        text = re.sub(pattern, repl, text)

    return text


def build_tree(text, loader_context=None):
    base_url = loader_context.get("base_url", None) if loader_context else None
    tree = lxml.html.fragment_fromstring(text, create_parent="div", base_url=base_url)

    if base_url:
        # Workaround for https://bugs.launchpad.net/lxml/+bug/1576598.
        # FIXME: Remove this when a workaround is released.
        tree.getroottree().docinfo.URL = base_url

    # Scrapy expects an iterator which it unpacks and feeds to the next
    # function in the pipeline. trees are iterators but we don't want to them
    # to get unpacked so we wrap the tree in another iterator.
    return [tree]


def serialize_tree(tree, in_make_links=False):
    return lxml.html.tostring(tree, encoding="unicode")


def make_links_absolute(tree):
    if tree.base_url:
        # Make references in tags like <a> and <img> absolute.
        tree.make_links_absolute(handle_failures="ignore")
    return [tree]


def cleanup_html(tree, loader_context):
    for elem_child, parent_dist in loader_context.get("pullup_elems", {}).items():
        selector = CSSSelector(elem_child)
        for elem in selector(tree):
            parent = elem
            for _ in range(parent_dist):
                parent = parent.getparent()
            if parent is not None and parent.getparent() is not None:
                parent.getparent().replace(parent, elem)
            else:
                logger.error(
                    'Could not find parent with distance {} for selector "{}".'.format(
                        parent_dist, elem_child
                    )
                )

    for elem_sel, elem_new in loader_context.get("replace_elems", {}).items():
        elem_new = lxml.html.fragment_fromstring(elem_new)
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            # New element could be replaced more than once but every node must be a
            # different element.
            elem.getparent().replace(elem, deepcopy(elem_new))

    remove_elems = []

    settings = get_feeds_settings()
    remove_images = settings.getbool("FEEDS_CONFIG_REMOVE_IMAGES")
    if remove_images:
        remove_elems += ["img"]

    # Remove tags.
    for elem_sel in loader_context.get("remove_elems", []) + remove_elems:
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.drop_tree()

    for elem_sel in loader_context.get("remove_elems_xpath", []):
        for elem in tree.xpath(elem_sel):
            elem.drop_tree()

    # Change tag names.
    for elem_sel, elem_tag in loader_context.get("change_tags", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.tag = elem_tag

    # tree.iter() iterates over the tree including the root node.
    for elem in tree.iter():
        # Remove class and id attribute from all elements which are not needed
        # in the feed.
        elem.attrib.pop("class", None)
        elem.attrib.pop("id", None)
        # Delete data- attributes that have no general meaning.
        for attrib in list(elem.attrib.keys()):
            if attrib.startswith("data-"):
                elem.attrib.pop(attrib)
<commit_msg>Add option to change attribute names.<commit_after>import html
import logging
import os
import re
from copy import deepcopy
from datetime import datetime
from textwrap import TextWrapper

import dateparser
import lxml
from dateutil.parser import parse as dateutil_parse
from dateutil.tz import gettz
from lxml import etree
from lxml.cssselect import CSSSelector
from lxml.html.clean import Cleaner
from scrapy.loader import ItemLoader
from scrapy.loader.processors import Compose, Identity, Join, MapCompose, TakeFirst
from w3lib.html import remove_tags

from feeds.items import FeedEntryItem, FeedItem
from feeds.settings import get_feeds_settings

logger = logging.getLogger(__name__)


def parse_datetime(date_time, loader_context):
    if isinstance(date_time, datetime):
        return date_time
    elif isinstance(date_time, str):
        try:
            return dateutil_parse(
                date_time.strip(),
                dayfirst=loader_context.get("dayfirst", False),
                yearfirst=loader_context.get("yearfirst", True),
                ignoretz=loader_context.get("ignoretz", False),
            )
        except ValueError:
            # If dateutil can't parse it, it might be a human-readable date.
            return dateparser.parse(date_time)
    else:
        raise ValueError("date_time must be datetime or a str.")


def apply_timezone(date_time, loader_context):
    if not date_time.tzinfo:
        # If date_time object is not aware, apply timezone from loader_context.
        # In case a timezone is not supplied, just assume UTC.
        date_time = date_time.replace(
            tzinfo=gettz(loader_context.get("timezone", "UTC"))
        )
    return date_time


def replace_regex(text, loader_context):
    for pattern, repl in loader_context.get("replace_regex", {}).items():
        text = re.sub(pattern, repl, text)

    return text


def build_tree(text, loader_context=None):
    base_url = loader_context.get("base_url", None) if loader_context else None
    tree = lxml.html.fragment_fromstring(text, create_parent="div", base_url=base_url)

    if base_url:
        # Workaround for https://bugs.launchpad.net/lxml/+bug/1576598.
        # FIXME: Remove this when a workaround is released.
        tree.getroottree().docinfo.URL = base_url

    # Scrapy expects an iterator which it unpacks and feeds to the next
    # function in the pipeline. trees are iterators but we don't want to them
    # to get unpacked so we wrap the tree in another iterator.
    return [tree]


def serialize_tree(tree, in_make_links=False):
    return lxml.html.tostring(tree, encoding="unicode")


def make_links_absolute(tree):
    if tree.base_url:
        # Make references in tags like <a> and <img> absolute.
        tree.make_links_absolute(handle_failures="ignore")
    return [tree]


def cleanup_html(tree, loader_context):
    for elem_child, parent_dist in loader_context.get("pullup_elems", {}).items():
        selector = CSSSelector(elem_child)
        for elem in selector(tree):
            parent = elem
            for _ in range(parent_dist):
                parent = parent.getparent()
            if parent is not None and parent.getparent() is not None:
                parent.getparent().replace(parent, elem)
            else:
                logger.error(
                    'Could not find parent with distance {} for selector "{}".'.format(
                        parent_dist, elem_child
                    )
                )

    for elem_sel, elem_new in loader_context.get("replace_elems", {}).items():
        elem_new = lxml.html.fragment_fromstring(elem_new)
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            # New element could be replaced more than once but every node must be a
            # different element.
            elem.getparent().replace(elem, deepcopy(elem_new))

    remove_elems = []

    settings = get_feeds_settings()
    remove_images = settings.getbool("FEEDS_CONFIG_REMOVE_IMAGES")
    if remove_images:
        remove_elems += ["img"]

    # Remove tags.
    for elem_sel in loader_context.get("remove_elems", []) + remove_elems:
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.drop_tree()

    for elem_sel in loader_context.get("remove_elems_xpath", []):
        for elem in tree.xpath(elem_sel):
            elem.drop_tree()

    # Change attrib names.
    for elem_sel, attribs in loader_context.get("change_attribs", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            for attrib in elem.attrib.keys():
                if attrib in attribs:
                    elem.attrib[attribs[attrib]] = elem.attrib.pop(attrib)

    # Change tag names.
    for elem_sel, elem_tag in loader_context.get("change_tags", {}).items():
        selector = CSSSelector(elem_sel)
        for elem in selector(tree):
            elem.tag = elem_tag

    # tree.iter() iterates over the tree including the root node.
    for elem in tree.iter():
        # Remove class and id attribute from all elements which are not needed
        # in the feed.
        elem.attrib.pop("class", None)
        elem.attrib.pop("id", None)
        # Delete data- attributes that have no general meaning.
        for attrib in list(elem.attrib.keys()):
            if attrib.startswith("data-"):
                elem.attrib.pop(attrib)

{'fraction_non_alphanumeric': 0.08667492142108772, 'fraction_numerical': 0.001523954662348795, 'mean_word_length': 2.6971830985915495, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1497575', 'n_tokens_mistral': 3022, 'n_tokens_neox': 2924, 'n_words': 888}