<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

__version__ = "0.0.3"

try:
    __FEEDFINDER2_SETUP__
except NameError:
    __FEEDFINDER2_SETUP__ = False

if not __FEEDFINDER2_SETUP__:
    __all__ = ["find_feeds"]

    import logging
    import requests
    from BeautifulSoup import BeautifulSoup
    from six.moves.urllib import parse as urlparse


def coerce_url(url):
    url = url.strip()
    if url.startswith("feed://"):
        return "http://{0}".format(url[7:])
    for proto in ["http://", "https://"]:
        if url.startswith(proto):
            return url
    return "http://{0}".format(url)


class FeedFinder(object):

    def __init__(self, user_agent=None):
        if user_agent is None:
            user_agent = "feedfinder2/{0}".format(__version__)
        self.user_agent = user_agent

    def get_feed(self, url):
        try:
            r = requests.get(url, headers={"User-Agent": self.user_agent})
        except Exception as e:
            logging.warn("Error while getting '{0}'".format(url))
            logging.warn("{0}".format(e))
            return None
        return r.text<commit_msg>Feedfinder user agent is presenting problems. Changing to 'NewsBlur Feed Finder'<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

__version__ = "0.0.3"

try:
    __FEEDFINDER2_SETUP__
except NameError:
    __FEEDFINDER2_SETUP__ = False

if not __FEEDFINDER2_SETUP__:
    __all__ = ["find_feeds"]

    import logging
    import requests
    from BeautifulSoup import BeautifulSoup
    from six.moves.urllib import parse as urlparse


def coerce_url(url):
    url = url.strip()
    if url.startswith("feed://"):
        return "http://{0}".format(url[7:])
    for proto in ["http://", "https://"]:
        if url.startswith(proto):
            return url
    return "http://{0}".format(url)


class FeedFinder(object):

    def __init__(self, user_agent=None):
        if user_agent is None:
            user_agent = "NewsBlur Feed Finder"
        self.user_agent = user_agent

    def get_feed(self, url, skip_user_agent=False):
        try:
            r = requests.get(url, headers={"User-Agent": self.user_agent if not skip_user_agent else None})
        except Exception as e:
            logging.warn("Error while getting '{0}'".format(url))
            logging.warn("{0}".format(e))
            return None
        if not skip_user_agent and r.status_code == 403:
            return self.get_feed(url, skip_user_agent=True)
        return r.text
{'fraction_non_alphanumeric': 0.12116504854368933, 'fraction_numerical': 0.011262135922330097, 'mean_word_length': 2.6695156695156697, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3997577', 'n_tokens_mistral': 856, 'n_tokens_neox': 837, 'n_words': 202}