<commit_before>import random, urllib, email
from xml.etree import ElementTree as ET
from datetime import datetime

from molly.conf.settings import batch

from molly.apps.podcasts.providers import BasePodcastsProvider
from molly.apps.podcasts.models import Podcast, PodcastItem, PodcastEnclosure

class RSSPodcastsProvider(BasePodcastsProvider):
    PODCAST_ATTRS = (
        ('guid', 'guid'),
        ('title', 'title'),
        ('author', '{itunes:}author'),
        ('duration', '{itunes:}duration'),
        ('published_date', 'pubDate'),
        ('description', 'description'),
        
#       ('itunesu_code', '{itunesu:}code'),
    )

    def __init__(self, podcasts):
        self.podcasts = podcasts
    
    @batch('%d * * * *' % random.randint(0, 59))
    def import_data(self, metadata, output):
        for slug, url in self.podcasts:
            podcast, url = Podcast.objects.get_or_create(
                provider=self.class_path,
                rss_url=url,
                defaults={'slug': slug})
            
            podcast.slug = slug
            self.update_podcast(podcast)
            
    def determine_license(self, o):
        license = o.find('{http://purl.org/dc/terms/}license') or \
                  o.find('{http://backend.userland.com/creativeCommonsRssModule}license')
        
        return license.text if license != None else None
        
    def update_podcast(self, podcast):
        def gct(node, name):
            try:
                value = node.find(name).text
                if name == 'pubDate':
                    value = datetime.fromtimestamp(
                        email.utils.mktime_tz(
                            email.utils.parsedate_tz(value)))
                elif name == '{itunes:}duration':
                    value = int(value)
                return value
            except AttributeError:
                return None

        xml = ET.parse(urllib.urlopen(podcast.rss_url))

        podcast.title = xml.find('.//channel/title').text
        podcast.description = xml.find('.//channel/description').text
        
        podcast.license = self.determine_license(xml.find('.//channel'))

        guids = []
        for item in xml.findall('.//channel/item'):
            if not gct(item, 'guid'):
                continue

            podcast_item, created = PodcastItem.objects.get_or_create(podcast=podcast, guid=gct(item, 'guid'))

            old_order = podcast_item.order
            try:
                podcast_item.order = int(item.find('{http://ns.ox.ac.uk/namespaces/oxitems/TopDownloads}position').text)
            except (AttributeError, TypeError):
                pass

            require_save = old_order != podcast_item.order
            for attr, x_attr in self.PODCAST_ATTRS:
                if getattr(podcast_item, attr) != gct(item, x_attr):
                    setattr(podcast_item, attr, gct(item, x_attr))
                    require_save = True
            license = self.determine_license(item)
            if require_save or podcast_item.license != license:
                podcast_item.license = license
                podcast_item.save()
<commit_msg>RSSPodcastsProvider now pulls logo from RSS feed.<commit_after>import random, urllib, email
from xml.etree import ElementTree as ET
from datetime import datetime

from molly.conf.settings import batch

from molly.apps.podcasts.providers import BasePodcastsProvider
from molly.apps.podcasts.models import Podcast, PodcastItem, PodcastEnclosure

class RSSPodcastsProvider(BasePodcastsProvider):
    PODCAST_ATTRS = (
        ('guid', 'guid'),
        ('title', 'title'),
        ('author', '{itunes:}author'),
        ('duration', '{http://www.itunes.com/dtds/podcast-1.0.dtd}duration'),
        ('published_date', 'pubDate'),
        ('description', 'description'),
        
#       ('itunesu_code', '{itunesu:}code'),
    )

    def __init__(self, podcasts):
        self.podcasts = podcasts
    
    @batch('%d * * * *' % random.randint(0, 59))
    def import_data(self, metadata, output):
        for slug, url in self.podcasts:
            podcast, url = Podcast.objects.get_or_create(
                provider=self.class_path,
                rss_url=url,
                defaults={'slug': slug})
            
            podcast.slug = slug
            self.update_podcast(podcast)
            
    def determine_license(self, o):
        license = o.find('{http://purl.org/dc/terms/}license') or \
                  o.find('{http://backend.userland.com/creativeCommonsRssModule}license')
        
        return license.text if license is not None else None
        
    def update_podcast(self, podcast):
        def gct(node, name):
            try:
                value = node.find(name).text
                if name == 'pubDate':
                    value = datetime.fromtimestamp(
                        email.utils.mktime_tz(
                            email.utils.parsedate_tz(value)))
                elif name == '{itunes:}duration':
                    value = int(value)
                return value
            except AttributeError:
                return None

        xml = ET.parse(urllib.urlopen(podcast.rss_url))

        podcast.title = xml.find('.//channel/title').text
        podcast.description = xml.find('.//channel/description').text
        
        podcast.license = self.determine_license(xml.find('.//channel'))

        logo = xml.find('.//channel/image/url')
        podcast.logo = logo.text if logo is not None else None

        guids = []
        for item in xml.findall('.//channel/item'):
            if not gct(item, 'guid'):
                continue

            podcast_item, created = PodcastItem.objects.get_or_create(podcast=podcast, guid=gct(item, 'guid'))

            old_order = podcast_item.order
            try:
                podcast_item.order = int(item.find('{http://ns.ox.ac.uk/namespaces/oxitems/TopDownloads}position').text)
            except (AttributeError, TypeError):
                pass

            require_save = old_order != podcast_item.order
            for attr, x_attr in self.PODCAST_ATTRS:
                if getattr(podcast_item, attr) != gct(item, x_attr):
                    setattr(podcast_item, attr, gct(item, x_attr))
                    require_save = True
            license = self.determine_license(item)
            if require_save or podcast_item.license != license:
                podcast_item.license = license
                podcast_item.save()

{'fraction_non_alphanumeric': 0.11433840479483633, 'fraction_numerical': 0.0012294452128477024, 'mean_word_length': 1.9210053859964094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '812782', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1808, 'n_words': 367}