<commit_before>"""Library for adding/removing feeds with Superfeedr's PubSubHubbub API"""
__version__ = '0.3.0'

import hashlib
import hmac
import httplib
import urllib

from xml.dom import minidom

import dateutil.parser
import feedparser


class Superfeedr(object):

    def __init__(self, superfeedr_username, superfeedr_password,
            user_agent=None):
        if user_agent is None:
            user_agent = "PubSubSuperfeedr/%s" % __version__
        self.superfeedr_username = superfeedr_username
        self.superfeedr_password = <PASSWORD>
        self.user_agent = user_agent

    def verify_feed_url(self, feed_url):
        """Verify that feed_url has feed items.

        Returns False if there aren't feed items, True if there are.

        """
        parsed_data = feedparser.parse(feed_url)
        if not parsed_data.entries:
            return False
        return True
<commit_msg>Use bozo to check feed validity, add verify_feed_url_with_message<commit_after>"""Library for adding/removing feeds with Superfeedr's PubSubHubbub API"""
__version__ = '0.3.1'

import hashlib
import hmac
import httplib
import urllib

from xml.dom import minidom

import dateutil.parser
import feedparser


class Superfeedr(object):

    def __init__(self, superfeedr_username, superfeedr_password,
            user_agent=None):
        if user_agent is None:
            user_agent = "PubSubSuperfeedr/%s" % __version__
        self.superfeedr_username = superfeedr_username
        self.superfeedr_password = <PASSWORD>
        self.user_agent = user_agent

    def verify_feed_url(self, feed_url):
        """Verify that feed_url appears to be valid.

        This checks for the feedparser 'bozo' flag.
        http://www.feedparser.org/docs/bozo.html

        """
        parsed_data = feedparser.parse(feed_url)
        return not parsed_data.bozo

    def verify_feed_url_with_message(self, feed_url):
        """Verify that feed_url appears to be valid, and returns the problem

        This checks for the feedparser 'bozo' flag, and returns a tuple of
        (valid, reason). reason will be None if valid is True.
        http://www.feedparser.org/docs/bozo.html

        """
        parsed_data = feedparser.parse(feed_url)
        if not parsed_data.bozo:
            return True, None
        exc = parsed_data.bozo_exception
        return False, "%s at line %d, column %d" % (
            exc.getMessage(), exc.getLineNumber(), exc.getColumnNumber())

{'fraction_non_alphanumeric': 0.07717171717171717, 'fraction_numerical': 0.0024242424242424242, 'mean_word_length': 2.893081761006289, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7107476', 'n_tokens_mistral': 746, 'n_tokens_neox': 733, 'n_words': 209}