<commit_before># -*- coding: utf-8 -*-


import logging
import re

from lxml.html import fromstring, HTMLParser, defs
from lxml.html.clean import Cleaner
from lxml.etree import XPathEvalError


logger = logging.getLogger(__name__)


class ContentPipeline(object):
    DEFAULT_ALLOW_CLASSES_NAME = 'allow_classes'
    DEFAULT_REMOVED_CLASSES_NAME = 'removed_classes'
    DEFAULT_REMOVED_XPATH_NODES_NAME = 'removed_xpath_nodes'
    DEFAULT_SAFE_ATTRS = 'safe_attrs'

    SAFE_ATTRS = {'style',
                  'float',
                  'src',
                  'font-size',
                  'font-family',
                  'align'}

    REMOVED_ATTRS = ('class',
                     'id',
                     'width',
                     'height')

    STYLE_REMOVED_ATTRS = (r'width\s*:\s*\d+[^;]*(;|$)',
                           r'height\s*:\s*\d+[^;]*(;|$)')

    @classmethod
    def from_settings(cls, settings):
        cls.ALLOW_CLASSES_NAME = settings.get(
            'SPIDER_ALLOW_CLASSES_NAME',
            cls.DEFAULT_ALLOW_CLASSES_NAME)
        cls.REMOVED_CLASSES_NAME = settings.get(
            'SPIDER_REMOVED_CLASSES_NAME',
            cls.DEFAULT_REMOVED_CLASSES_NAME)
        cls.REMOVED_XPATH_NODES_NAME = settings.get(
            'SPIDER_REMOVED_XPATH_NODES_NAME',
            cls.DEFAULT_REMOVED_XPATH_NODES_NAME)
        cls.SAFE_ATTRS_NAME = settings.get(
            'SPIDER_SAFE_ATTRS_NAME',
            cls.DEFAULT_SAFE_ATTRS)
        return cls()

    def format_title(self, title):
        return re.sub(r'(\r|\n|\s)+',
                      ' ',
                      title)

    def make_abs_link(self, doc, link):
        doc.make_links_absolute(link)
        return doc

    def remove_element_with_class(self, doc, removed_classes):
        for e in doc.xpath('//div[@class]'):
            if any(cls in e.get('class').lower() for cls in removed_classes):
                e.drop_tree()
        return doc

    def remove_element_with_xpath(self, doc, removed_xpath_nodes):
        for xpath in filter(lambda x: x.strip(),
                            removed_xpath_nodes):
            try:
                nodes = doc.xpath(xpath)
            except XPathEvalError:
                logger.error((
                    'Error in pipeline content invalid xpath[{}]'
                    ).format(xpath))
            else:
                for _ in nodes:
                    _.drop_tree()
        return doc

    def clean_html(self, doc, allow_classes=None, safe_attrs=None):
        allow_classes = allow_classes or ()
        safe_attrs = (set(defs.safe_attrs) |
                      self.SAFE_ATTRS |
                      set(safe_attrs or []))
        cleaner = Cleaner(safe_attrs_only=True,
                          safe_attrs=safe_attrs)
        doc = cleaner.clean_html(doc)

        while (len(doc) == 1):
            doc = doc[0]

        def rename_tag(doc):
            for e in doc.iter():
                if e.tag.lower() == 'article':
                    e.tag = 'div'

        rename_tag(doc)

        def remove_attr(doc):
            pattern = re.compile('|'.join(self.STYLE_REMOVED_ATTRS),
                                 flags=re.IGNORECASE)
            for e in doc.iter():
                if 'style' in e.attrib:
                    style_ = re.sub(pattern,
                                    '',
                                    e.get('style')).strip()
                    style = re.sub(r'\s{2,}',
                                   ' ',
                                   style_).strip()
                    if style:
                        e.attrib['style'] = style
                    else:
                        e.attrib.pop('style')
                for attr in e.attrib:
                    if (attr in self.REMOVED_ATTRS and
                        not (attr == 'class' and
                             e.get(attr).strip() in allow_classes)):
                        e.attrib.pop(attr)

        remove_attr(doc)

        def remove_tags(doc):
            TAGS = ('//i[not(text())]',
                    '//ins[not(text())]')

            for t in TAGS:
                for e in doc.xpath(t):
                    e.drop_tree()

            for e in doc.xpath('//a/div'):
                e.drop_tree()

            for e in doc.xpath('//div[not(.//div) and not(.//img)'):
                text = e.text_content().strip(' \r\n\t')
                if not text:
                    e.drop_tree()

        remove_tags(doc)

        return doc

    def process_item(self, item, spider):
        item['title'] = self.format_title(item['title'])
        doc = item['content']
        if isinstance(doc,
                      (str, bytes)):
            doc = fromstring(bytes(bytearray(doc,
                                             encoding=item['encoding'])),
                             parser=HTMLParser(encoding=item['encoding']))

        # remove element with class name for clean display<commit_msg>xpath<commit_after># -*- coding: utf-8 -*-


import logging
import re

from lxml.html import fromstring, HTMLParser, defs
from lxml.html.clean import Cleaner
from lxml.etree import XPathEvalError


logger = logging.getLogger(__name__)


class ContentPipeline(object):
    DEFAULT_ALLOW_CLASSES_NAME = 'allow_classes'
    DEFAULT_REMOVED_CLASSES_NAME = 'removed_classes'
    DEFAULT_REMOVED_XPATH_NODES_NAME = 'removed_xpath_nodes'
    DEFAULT_SAFE_ATTRS = 'safe_attrs'

    SAFE_ATTRS = {'style',
                  'float',
                  'src',
                  'font-size',
                  'font-family',
                  'align'}

    REMOVED_ATTRS = ('class',
                     'id',
                     'width',
                     'height')

    STYLE_REMOVED_ATTRS = (r'width\s*:\s*\d+[^;]*(;|$)',
                           r'height\s*:\s*\d+[^;]*(;|$)')

    @classmethod
    def from_settings(cls, settings):
        cls.ALLOW_CLASSES_NAME = settings.get(
            'SPIDER_ALLOW_CLASSES_NAME',
            cls.DEFAULT_ALLOW_CLASSES_NAME)
        cls.REMOVED_CLASSES_NAME = settings.get(
            'SPIDER_REMOVED_CLASSES_NAME',
            cls.DEFAULT_REMOVED_CLASSES_NAME)
        cls.REMOVED_XPATH_NODES_NAME = settings.get(
            'SPIDER_REMOVED_XPATH_NODES_NAME',
            cls.DEFAULT_REMOVED_XPATH_NODES_NAME)
        cls.SAFE_ATTRS_NAME = settings.get(
            'SPIDER_SAFE_ATTRS_NAME',
            cls.DEFAULT_SAFE_ATTRS)
        return cls()

    def format_title(self, title):
        return re.sub(r'(\r|\n|\s)+',
                      ' ',
                      title)

    def make_abs_link(self, doc, link):
        doc.make_links_absolute(link)
        return doc

    def remove_element_with_class(self, doc, removed_classes):
        for e in doc.xpath('//div[@class]'):
            if any(cls in e.get('class').lower() for cls in removed_classes):
                e.drop_tree()
        return doc

    def remove_element_with_xpath(self, doc, removed_xpath_nodes):
        for xpath in filter(lambda x: x.strip(),
                            removed_xpath_nodes):
            try:
                nodes = doc.xpath(xpath)
            except XPathEvalError:
                logger.error((
                    'Error in pipeline content invalid xpath[{}]'
                    ).format(xpath))
            else:
                for _ in nodes:
                    _.drop_tree()
        return doc

    def clean_html(self, doc, allow_classes=None, safe_attrs=None):
        allow_classes = allow_classes or ()
        safe_attrs = (set(defs.safe_attrs) |
                      self.SAFE_ATTRS |
                      set(safe_attrs or []))
        cleaner = Cleaner(safe_attrs_only=True,
                          safe_attrs=safe_attrs)
        doc = cleaner.clean_html(doc)

        while (len(doc) == 1):
            doc = doc[0]

        def rename_tag(doc):
            for e in doc.iter():
                if e.tag.lower() == 'article':
                    e.tag = 'div'

        rename_tag(doc)

        def remove_attr(doc):
            pattern = re.compile('|'.join(self.STYLE_REMOVED_ATTRS),
                                 flags=re.IGNORECASE)
            for e in doc.iter():
                if 'style' in e.attrib:
                    style_ = re.sub(pattern,
                                    '',
                                    e.get('style')).strip()
                    style = re.sub(r'\s{2,}',
                                   ' ',
                                   style_).strip()
                    if style:
                        e.attrib['style'] = style
                    else:
                        e.attrib.pop('style')
                for attr in e.attrib:
                    if (attr in self.REMOVED_ATTRS and
                        not (attr == 'class' and
                             e.get(attr).strip() in allow_classes)):
                        e.attrib.pop(attr)

        remove_attr(doc)

        def remove_tags(doc):
            TAGS = ('//i[not(text())]',
                    '//ins[not(text())]')

            for t in TAGS:
                for e in doc.xpath(t):
                    e.drop_tree()

            for e in doc.xpath('//a/div'):
                e.drop_tree()

            for e in doc.xpath('//div[not(descendant-or-self::div) and '
                               'not(descendant-or-self::img)'):
                text = e.text_content().strip(' \r\n\t')
                if not text:
                    e.drop_tree()

        remove_tags(doc)

        return doc

    def process_item(self, item, spider):
        item['title'] = self.format_title(item['title'])
        doc = item['content']
        if isinstance(doc,
                      (str, bytes)):
            doc = fromstring(bytes(bytearray(doc,
                                             encoding=item['encoding'])),
                             parser=HTMLParser(encoding=item['encoding']))

        # remove element with class name for clean display
{'fraction_non_alphanumeric': 0.11077844311377245, 'fraction_numerical': 0.0007984031936127744, 'mean_word_length': 1.3540051679586564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '104879', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2838, 'n_words': 539}