<commit_before>import feedparser

from vidscraper.bulk_import import opensearch, blip, vimeo

IMPORTERS = (
    opensearch,
    blip,
    vimeo,
    )

def bulk_import(feed_url, parsed_feed=None):
    """
    Takes a URL for a feed, and returns a feedparser instance with the entries
    filled in with all of the videos from that feed.  It'll take care of paging
    through videos from services like blip.tv and Vimeo or services which
    support OpenSearch (like YouTube).
    """
    if parsed_feed is None:
        parsed_feed = feedparser.parse(feed_url)
    for importer in IMPORTERS:
        if importer.video_count(parsed_feed) is not None:
            return importer.bulk_import(parsed_feed)

    # if we can't figure out how to bulk import the feed, just return the items
    # in the feed
    return parsed_feed


def video_count(feed_url, parsed_feed=None):
    """
    Takes a URL for a feed, and returns the number of videos we think we can
    import from it.
    """
    if parsed_feed is None:
        parsed_feed = feedparser.parse(feed_url)
    for importer in IMPORTERS:
        count = importer.video_count(parsed_feed)
        if count is not None:
            return count

    return len(parsed_feed.entries)<commit_msg>Bug #12334: Add a Feed page doesn't check if items are actually videos<commit_after>import feedparser

from vidscraper.bulk_import import opensearch, blip, vimeo
from vidscraper import miroguide_util

IMPORTERS = (
    opensearch,
    blip,
    vimeo,
    )

def bulk_import(feed_url, parsed_feed=None):
    """
    Takes a URL for a feed, and returns a feedparser instance with the entries
    filled in with all of the videos from that feed.  It'll take care of paging
    through videos from services like blip.tv and Vimeo or services which
    support OpenSearch (like YouTube).
    """
    if parsed_feed is None:
        parsed_feed = feedparser.parse(feed_url)
    for importer in IMPORTERS:
        if importer.video_count(parsed_feed) is not None:
            return importer.bulk_import(parsed_feed)

    # if we can't figure out how to bulk import the feed, just return the items
    # in the feed
    return parsed_feed


def video_count(feed_url, parsed_feed=None):
    """
    Takes a URL for a feed, and returns the number of videos we think we can
    import from it.
    """
    if parsed_feed is None:
        parsed_feed = feedparser.parse(feed_url)
    for importer in IMPORTERS:
        count = importer.video_count(parsed_feed)
        if count is not None:
            return count

    count = 0
    for entry in parsed_feed.entries:
        if miroguide_util.get_first_video_enclosure(entry) is not None:
            count += 1
    return count
{'fraction_non_alphanumeric': 0.05778432094221568, 'fraction_numerical': 0.002576370997423629, 'mean_word_length': 2.8662873399715507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3812269', 'n_tokens_mistral': 827, 'n_tokens_neox': 824, 'n_words': 306}