<commit_before>import html
import urllib
import logging
import requests
import feedparser
from conf import USER_AGENT
from bs4 import BeautifulSoup, SoupStrainer

from web.lib.utils import try_keys, try_get_icon_url, rebuild_url

logger = logging.getLogger(__name__)
logging.captureWarnings(True)
ACCEPTED_MIMETYPES = ('application/rss+xml', 'application/rdf+xml',
                      'application/atom+xml', 'application/xml', 'text/xml')


def is_parsing_ok(parsed_feed):
    return parsed_feed['entries'] or not parsed_feed['bozo']


def escape_keys(*keys):
    def wrapper(func):
        def metawrapper(*args, **kwargs):
            result = func(*args, **kwargs)
            for key in keys:
                if key in result:
                    result[key] = html.unescape(result[key])
            return result
        return metawrapper<commit_msg>fix: value might be None<commit_after>import html
import urllib
import logging
import requests
import feedparser
from conf import USER_AGENT
from bs4 import BeautifulSoup, SoupStrainer

from web.lib.utils import try_keys, try_get_icon_url, rebuild_url

logger = logging.getLogger(__name__)
logging.captureWarnings(True)
ACCEPTED_MIMETYPES = ('application/rss+xml', 'application/rdf+xml',
                      'application/atom+xml', 'application/xml', 'text/xml')


def is_parsing_ok(parsed_feed):
    return parsed_feed['entries'] or not parsed_feed['bozo']


def escape_keys(*keys):
    def wrapper(func):
        def metawrapper(*args, **kwargs):
            result = func(*args, **kwargs)
            for key in keys:
                if result.get(key):
                    result[key] = html.unescape(result[key])
            return result
        return metawrapper
{'fraction_non_alphanumeric': 0.09705372616984402, 'fraction_numerical': 0.0011554015020219526, 'mean_word_length': 3.2450980392156863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '485162', 'n_tokens_mistral': 512, 'n_tokens_neox': 505, 'n_words': 131}