<commit_before>import random, urllib, email
from lxml import etree
from datetime import datetime

from molly.conf.settings import batch
import dateutil.parser

from molly.apps.podcasts.providers import BasePodcastsProvider
from molly.apps.podcasts.models import Podcast, PodcastItem, PodcastEnclosure

class Namespace(object):
    def __init__(self, ns):
        self.ns = ns
    def __call__(self, local):
        return '{%s}%s' % (self.ns, local)

class RSSPodcastsProvider(BasePodcastsProvider):
    @property
    def PODCAST_ATTRS(self):
        atom = self.atom
        return (
            ('guid', ('guid', atom('id'),)),
            ('title', ('title', atom('title'),)),
            ('author', ('{itunes:}author',)),
            ('duration', ('{itunes:}duration',)),
            ('published_date', ('pubDate', atom('published'),)),
            ('description', ('description', atom('summary'),)),
    #       ('itunesu_code', '{itunesu:}code'),
        )

    def __init__(self, podcasts, medium=None):
        self.podcasts = podcasts
        self.medium = medium
    
    @property
    def atom(self):
        return Namespace('http://www.w3.org/2005/Atom')
    
    @batch('%d * * * *' % random.randint(0, 59))
    def import_data(self, metadata, output):
        for slug, url in self.podcasts:
            podcast, url = Podcast.objects.get_or_create(
                provider=self.class_path,
                rss_url=url,
                defaults={'slug': slug})
            if self.medium: 
                podcast.medium = self.medium
                
            podcast.slug = slug
            self.update_podcast(podcast)
            
    def determine_license(self, o):
        license = o.find('{http://purl.org/dc/terms/}license') or \
                  o.find('{http://backend.userland.com/creativeCommonsRssModule}license')
        
        return getattr(license, 'text', None)
        
    def update_podcast(self, podcast):
        atom = self.atom
        def gct(node, names):
            for name in names:
                if node.find(name) is None:
                    continue
                value = node.find(name).text
                if name == 'pubDate':
                    value = datetime.fromtimestamp(
                        email.utils.mktime_tz(
                            email.utils.parsedate_tz(value)))
                elif name == atom('published'):
                    value = dateutil.parser.parse(value)
                elif name == '{itunes:}duration':
                    value = int(value)
                return value
            return None

        xml = etree.parse(urllib.urlopen(podcast.rss_url)).getroot()

        try:
            podcast.title = xml.find('.//channel/title').text
            podcast.description = xml.find('.//channel/description').text
        except AttributeError:
            podcast.title = xml.find(atom('title')).text
            podcast.description = xml.find(atom('subtitle')).text
        
        podcast.license = self.determine_license(xml.find('.//channel'))
        if self.medium is not None:
            podcast.medium = medium

        logo = xml.find('.//channel/image/url')
        podcast.logo = logo.text if logo is not None else None

        ids = []
        for item in xml.findall('.//channel/item') or xml.findall(atom('entry')):
            id = gct(item, ('guid', atom('id'),))
            if not id:
                continue
            
            podcast_item, created = PodcastItem.objects.get_or_create(podcast=podcast, guid=id)

            old_order = podcast_item.order
            try:
                podcast_item.order = int(item.find('{http://ns.ox.ac.uk/namespaces/oxitems/TopDownloads}position').text)<commit_msg>Fix errors from multiple podcasts being in the database<commit_after>import random, urllib, email
from lxml import etree
from datetime import datetime

from molly.conf.settings import batch
import dateutil.parser

from molly.apps.podcasts.providers import BasePodcastsProvider
from molly.apps.podcasts.models import Podcast, PodcastItem, PodcastEnclosure

class Namespace(object):
    def __init__(self, ns):
        self.ns = ns
    def __call__(self, local):
        return '{%s}%s' % (self.ns, local)

class RSSPodcastsProvider(BasePodcastsProvider):
    @property
    def PODCAST_ATTRS(self):
        atom = self.atom
        return (
            ('guid', ('guid', atom('id'),)),
            ('title', ('title', atom('title'),)),
            ('author', ('{itunes:}author',)),
            ('duration', ('{itunes:}duration',)),
            ('published_date', ('pubDate', atom('published'),)),
            ('description', ('description', atom('summary'),)),
    #       ('itunesu_code', '{itunesu:}code'),
        )

    def __init__(self, podcasts, medium=None):
        self.podcasts = podcasts
        self.medium = medium
    
    @property
    def atom(self):
        return Namespace('http://www.w3.org/2005/Atom')
    
    @batch('%d * * * *' % random.randint(0, 59))
    def import_data(self, metadata, output):
        for slug, url in self.podcasts:
            podcast, url = Podcast.objects.get_or_create(
                provider=self.class_path,
                rss_url=url,
                defaults={'slug': slug})
            if self.medium: 
                podcast.medium = self.medium
                
            podcast.slug = slug
            self.update_podcast(podcast)
            
    def determine_license(self, o):
        license = o.find('{http://purl.org/dc/terms/}license') or \
                  o.find('{http://backend.userland.com/creativeCommonsRssModule}license')
        
        return getattr(license, 'text', None)
        
    def update_podcast(self, podcast):
        atom = self.atom
        def gct(node, names):
            for name in names:
                if node.find(name) is None:
                    continue
                value = node.find(name).text
                if name == 'pubDate':
                    value = datetime.fromtimestamp(
                        email.utils.mktime_tz(
                            email.utils.parsedate_tz(value)))
                elif name == atom('published'):
                    value = dateutil.parser.parse(value)
                elif name == '{itunes:}duration':
                    value = int(value)
                return value
            return None

        xml = etree.parse(urllib.urlopen(podcast.rss_url)).getroot()

        try:
            podcast.title = xml.find('.//channel/title').text
            podcast.description = xml.find('.//channel/description').text
        except AttributeError:
            podcast.title = xml.find(atom('title')).text
            podcast.description = xml.find(atom('subtitle')).text
        
        podcast.license = self.determine_license(xml.find('.//channel'))
        if self.medium is not None:
            podcast.medium = medium

        logo = xml.find('.//channel/image/url')
        podcast.logo = logo.text if logo is not None else None

        ids = []
        for item in xml.findall('.//channel/item') or xml.findall(atom('entry')):
            id = gct(item, ('guid', atom('id'),))
            if not id:
                continue
            
            try:
                podcast_item, created = PodcastItem.objects.get_or_create(podcast=podcast, guid=id)
            except PodcastItem.MultipleObjectsReturned:
                PodcastItem.objects.filter(podcast=podcast, guid=id).delete()
                podcast_item, created = PodcastItem.objects.get_or_create(podcast=podcast, guid=id)

            old_order = podcast_item.order
            try:
                podcast_item.order = int(item.find('{http://ns.ox.ac.uk/namespaces/oxitems/TopDownloads}position').text)
{'fraction_non_alphanumeric': 0.1275228178429104, 'fraction_numerical': 0.002056819642627587, 'mean_word_length': 1.9040686823441582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '112222', 'n_tokens_mistral': 2209, 'n_tokens_neox': 2161, 'n_words': 446}