<commit_before># -*- coding: utf-8 -*-


import re

from lxml.html import fromstring, HTMLParser, defs
from lxml.html.clean import Cleaner
from lxml.etree import XPathEvalError

from ..log import logger


class ContentPipeline(object):
    DEFAULT_ALLOW_CLASSES_NAME = 'allow_classes'
    DEFAULT_REMOVED_CLASSES_NAME = 'removed_classes'
    DEFAULT_REMOVED_XPATH_NODES_NAME = 'removed_xpath_nodes'
    DEFAULT_SAFE_ATTRS = 'safe_attrs'

    SAFE_ATTRS = {'style',
                  'float',
                  'src',
                  'font-size',
                  'font-family',
                  'align'}

    REMOVED_ATTRS = ('class',
                     'id',
                     'width',
                     'height')

    STYLE_REMOVED_ATTRS = (r'width\s*=\s*"?\d+[^" ]*"?',
                           r'width\s*:\s*\d+[^;]*;',
                           r'height\s*=\s*"?\d+[^" ]*"?',
                           r'height\s*:\s*\d+[^;]*;')

    @classmethod
    def from_settings(cls, settings):
        cls.ALLOW_CLASSES_NAME = settings.get(<commit_msg>update style attribute regex<commit_after># -*- coding: utf-8 -*-


import re

from lxml.html import fromstring, HTMLParser, defs
from lxml.html.clean import Cleaner
from lxml.etree import XPathEvalError

from ..log import logger


class ContentPipeline(object):
    DEFAULT_ALLOW_CLASSES_NAME = 'allow_classes'
    DEFAULT_REMOVED_CLASSES_NAME = 'removed_classes'
    DEFAULT_REMOVED_XPATH_NODES_NAME = 'removed_xpath_nodes'
    DEFAULT_SAFE_ATTRS = 'safe_attrs'

    SAFE_ATTRS = {'style',
                  'float',
                  'src',
                  'font-size',
                  'font-family',
                  'align'}

    REMOVED_ATTRS = ('class',
                     'id',
                     'width',
                     'height')

    STYLE_REMOVED_ATTRS = (r'width\s*:\s*\d+[^;]*(;|$)',
                           r'height\s*:\s*\d+[^;]*(;|$)')

    @classmethod
    def from_settings(cls, settings):
        cls.ALLOW_CLASSES_NAME = settings.get(
{'fraction_non_alphanumeric': 0.13457760314341846, 'fraction_numerical': 0.0009823182711198428, 'mean_word_length': 1.8934659090909092, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6253189', 'n_tokens_mistral': 692, 'n_tokens_neox': 646, 'n_words': 112}