<commit_before>import datetime
import math
import re

import feedparser
import simplejson

from vidscraper.util import open_url_while_lying_about_agent

USERNAME_RE = re.compile('http://vimeo\.com/(\w+)/')

_cached_video_count = {}

def video_count(parsed_feed):
    if parsed_feed.feed.generator != 'The Vimeo':
        return None
    username = USERNAME_RE.search(parsed_feed.feed.link).group(1)
    json_data = simplejson.load(open_url_while_lying_about_agent(
            'http://vimeo.com/api/v2/%s/info.json' % username))
    _cached_video_count[parsed_feed.feed.link] = count = \
        json_data['total_videos_uploaded']
    return count


def bulk_import(parsed_feed):
    username = USERNAME_RE.search(parsed_feed.feed.link).group(1)
    count = _cached_video_count[parsed_feed.feed.link]
    post_url = 'http://vimeo.com/api/v2/%s/videos.json?page=%%i' % username
    parsed_feed = feedparser.FeedParserDict(parsed_feed.copy())
    parsed_feed.entries = []
    for i in range(1, int(math.ceil(count / 20.0)) + 1):
        json_data = simplejson.load(open_url_while_lying_about_agent(
                post_url % i))
        for video in json_data:
            parsed_feed.entries.append(
                feedparser.FeedParserDict(_json_to_feedparser(video)))

    # clean up cache
    if parsed_feed.feed.link in _cached_video_count:
        del _cached_video_count[parsed_feed.feed.link]

    return parsed_feed

def safe_decode(str_or_unicode):
    if isinstance(str_or_unicode, unicode):
        return str_or_unicode
    else:
        return str_or_unicode.decode('utf8')

def _json_to_feedparser(json):
    upload_date = datetime.datetime.strptime(
        json['upload_date'],
        '%Y-%m-%d %H:%M:%S')
    tags = [{'label': u'Tags',
             'scheme': None,
             'term': safe_decode(json['tags'])}]
    tags.extend({'label': None,<commit_msg>wrap dicts in FeedParserDicts recursively<commit_after>import datetime
import math
import re

import feedparser
import simplejson

from vidscraper.util import open_url_while_lying_about_agent

USERNAME_RE = re.compile('http://vimeo\.com/(\w+)/')

_cached_video_count = {}

def video_count(parsed_feed):
    if parsed_feed.feed.generator != 'The Vimeo':
        return None
    username = USERNAME_RE.search(parsed_feed.feed.link).group(1)
    json_data = simplejson.load(open_url_while_lying_about_agent(
            'http://vimeo.com/api/v2/%s/info.json' % username))
    _cached_video_count[parsed_feed.feed.link] = count = \
        json_data['total_videos_uploaded']
    return count


def bulk_import(parsed_feed):
    username = USERNAME_RE.search(parsed_feed.feed.link).group(1)
    count = _cached_video_count[parsed_feed.feed.link]
    post_url = 'http://vimeo.com/api/v2/%s/videos.json?page=%%i' % username
    parsed_feed = feedparser.FeedParserDict(parsed_feed.copy())
    parsed_feed.entries = []
    for i in range(1, int(math.ceil(count / 20.0)) + 1):
        json_data = simplejson.load(open_url_while_lying_about_agent(
                post_url % i))
        for video in json_data:
            parsed_feed.entries.append(feedparser_dict(
                    _json_to_feedparser(video)))

    # clean up cache
    if parsed_feed.feed.link in _cached_video_count:
        del _cached_video_count[parsed_feed.feed.link]

    return parsed_feed

def feedparser_dict(obj):
    if isinstance(obj, dict):
        return feedparser.FeedParserDict(dict(
                [(key, feedparser_dict(value))
                 for (key, value) in obj.items()]))
    if isinstance(obj, (list, tuple)):
        return [feedparser_dict(member) for member in obj]
    return obj

def safe_decode(str_or_unicode):
    if isinstance(str_or_unicode, unicode):
        return str_or_unicode
    else:
        return str_or_unicode.decode('utf8')

def _json_to_feedparser(json):
    upload_date = datetime.datetime.strptime(
        json['upload_date'],
        '%Y-%m-%d %H:%M:%S')
    tags = [{'label': u'Tags',
             'scheme': None,
             'term': safe_decode(json['tags'])}]
    tags.extend({'label': None,
{'fraction_non_alphanumeric': 0.11858190709046455, 'fraction_numerical': 0.004889975550122249, 'mean_word_length': 3.297268907563025, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1761801', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1375, 'n_words': 233}