<commit_before>from bs4 import BeautifulSoup
import urllib2
import re


_image_finder = re.compile(r"img src='(.*)'")

_statuses = {
    'Yes': 'used',
    'Using': 'using',
    'No': 'unused',
    'used': 'Yes',
    'using': 'Using',
    'unused': 'No',
    '': 'unused'
}

_status = {
    'books': {
        'wanted': ('wanted',),
        'owned': ('owned',),
        'used': ('read',),
        'all': ('owned', 'wanted', 'read')
    },
    'movies': {
        'wanted': ('wanted',),
        'used': ('watched',),
        'all': ('wanted', 'watched')
    }
}


def guess_attributes(item):
    return {
        'title': item.find('title').text,
        'image': _image_finder.search(item.find('description').text).groups()[0],
        'status': _statuses[item.find('listal:used').text],
        'since': item.find('pubdate').text,
        'url': item.find('link').text,
    }


def _get_items(url):
    data = BeautifulSoup(urllib2.urlopen(url).read())
    items = data.rss.find_all('item')
    return [guess_attributes(item) for item in items]


def _get_data(user, collection, ownership, status):
    base_url = 'http://%s.listal.com/rss/%s/%s/1?sortby=dateadded-desc'  # TODO: Handle paging
    items = []
    used_query = ('&used=%s' % _statuses[status] ) if status != 'all' else ''
    for s in _status[collection][ownership]:
        items.extend(_get_items((base_url % (user, collection, s)) + used_query))
    return items<commit_msg>set BS parser explicitly<commit_after>from bs4 import BeautifulSoup
import urllib2
import re


_image_finder = re.compile(r"img src='(.*)'")

_statuses = {
    'Yes': 'used',
    'Using': 'using',
    'No': 'unused',
    'used': 'Yes',
    'using': 'Using',
    'unused': 'No',
    '': 'unused'
}

_status = {
    'books': {
        'wanted': ('wanted',),
        'owned': ('owned',),
        'used': ('read',),
        'all': ('owned', 'wanted', 'read')
    },
    'movies': {
        'wanted': ('wanted',),
        'used': ('watched',),
        'all': ('wanted', 'watched')
    }
}


def guess_attributes(item):
    return {
        'title': item.find('title').text,
        'image': _image_finder.search(item.find('description').text).groups()[0],
        'status': _statuses[item.find('listal:used').text],
        'since': item.find('pubdate').text,
        'url': item.find('link').text,
    }


def _get_items(url):
    data = BeautifulSoup(urllib2.urlopen(url).read(), "html.parser")
    items = data.rss.find_all('item')
    return [guess_attributes(item) for item in items]


def _get_data(user, collection, ownership, status):
    base_url = 'http://%s.listal.com/rss/%s/%s/1?sortby=dateadded-desc'  # TODO: Handle paging
    items = []
    used_query = ('&used=%s' % _statuses[status] ) if status != 'all' else ''
    for s in _status[collection][ownership]:
        items.extend(_get_items((base_url % (user, collection, s)) + used_query))
    return items
{'fraction_non_alphanumeric': 0.20398351648351648, 'fraction_numerical': 0.003434065934065934, 'mean_word_length': 3.102816901408451, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1481011', 'n_tokens_mistral': 1031, 'n_tokens_neox': 1014, 'n_words': 201}