<commit_before>#!/usr/bin/env python

import feedparser
import time
import hashlib
import os
from os.path import expanduser
import logging
import re
import cgi
import requests
import argparse
from datetime import datetime as dt
from clint.textui import progress
from sqlobject import SQLObject, sqlite, DateTimeCol, UnicodeCol

parser = argparse.ArgumentParser(description='Download podcasts via the command line.')
parser.add_argument('--no-download', action='store_true',
                   help="Don't download any files. Just mark as read.")
parser.add_argument('--podcastdir', '-p', default='~/Downloads/podcasts', 
    help="Folder to download podcast files to.")
parser.add_argument('--basedir', '-b', default='/.upodder', 
    help="Folder to store subscriptions and seen database.")
parser.add_argument('--oldness', '-o', default=30, 
    help="Skip entries older than X days.")
parser.add_argument('--mark-seen', action='store_false',
    help="Just mark all entries as seen and exit.")
parser.add_argument('--log-level', '-l', default=5, 
    help="Logging level. 5=Debug.")
args = parser.parse_args()

YES = [1,"1","on","yes","Yes","YES","y","Y","true","True","TRUE","t","T"]
CONFIGCOMMENT = ['#',';','$',':','"',"'"]
BADFNCHARS = re.compile('[^\w]+',re.LOCALE)
TEMPDIR = '/tmp/upodder'
FILENAME = '%(entry_title)s.mp3'

# Initializing logging
l = logging.Logger('upodder', logging.DEBUG)
stderrHandler = logging.StreamHandler()
stderrHandler.setFormatter(logging.Formatter('%(message)s'))
l.addHandler(stderrHandler)

class SeenEntry(SQLObject):
    "Represents a single feed item, seen before. Used to keep track of download status."
    hashed = UnicodeCol()
    pub_date = DateTimeCol()

class EntryProcessor(object):
    "Processes single feed entry"
    def __init__(self, entry, feed):
        self.hashed = hashlib.sha1(entry['title'].encode('ascii', 'ignore')).hexdigest()
        self.pub_date = dt.fromtimestamp(time.mktime(entry.published_parsed))

        if args.mark_seen:
            SeenEntry(pub_date=self.pub_date, hashed=self.hashed)
            return

        # Let's check if we worked on this entry earlier...
        if SeenEntry.select(SeenEntry.q.hashed == self.hashed).count() > 0:
            l.debug("Already seen: %s"%(entry['title']))
            return
        
        # Let's check the entry's date
        if (dt.now() - self.pub_date).days > args.oldness:
            l.debug("Too old for us: %s"%entry['title'])
            return

        # Search for mpeg enclosures
        for enclosure in filter(lambda x: x.get('type') == 'audio/mpeg',entry.get('enclosures',[])):
            # Work only with first found audio/mpeg enclosure (Bad Thing? maybe :( )
            if self._download_enclosure(enclosure, entry, feed, args.no_download):
                SeenEntry( pub_date=self.pub_date, hashed=self.hashed)
            break

    def _download_enclosure(self, enclosure, entry, feed, no_download=False):
        """Performs downloading of specified file. Returns True on success and False in other case"""

        downloadto = TEMPDIR + os.sep + self.hashed

        if no_download:
            l.debug("Would download %s"%enclosure['href'])
            return True

        try:
            """Downloads URL to file, returns file name of download (from URL or Content-Disposition)"""
            if not os.path.exists(os.path.dirname(downloadto)):
                os.makedirs(os.path.dirname(file),0700)

            l.debug("Downloading %s"%enclosure['href'])
            r = requests.get(enclosure['href'], stream=True)
            with open(downloadto, 'wb') as f:
                if 'content-length' in r.headers:
                    total_length = int(r.headers['content-length'])
                    # print total_length, enclosure['length'] #TODO: which measure is more reliable?
                    r_iter = progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1)
                else:
                    r_iter = r.iter_content(chunk_size=1024)
                for chunk in r_iter: 
                    if chunk:
                        f.write(chunk)
                        f.flush()

            filename = cgi.parse_header(r.headers.get('content-disposition'))[1]['filename']
            if not filename:
                filename = "Untitled.mp3"

        except KeyboardInterrupt:
            l.info("Download aborted by Ctrl+c")
            try:
                user_wish = raw_input("Do you like to mark item as read? (y/N): ")
                if user_wish in YES:
                    return True
            except KeyboardInterrupt:
                print "No"
            return False

        # Move downloaded file to its final destination
        moveto = args.podcastsdir + os.sep + self._generate_filename(filename, entry, feed)
        l.debug("Moving {%s} to {%s}"%(downloadto,moveto))
        if not os.path.exists(os.path.dirname(moveto)): os.makedirs(os.path.dirname(moveto),0750)
        os.rename(downloadto,moveto)
        return True

    def _generate_filename(self, filename, entry, feed):
        """Generates file name for this enclosure based on config settins"""
        (year,month,day,hour,minute,second,weekday,yearday,leap) = time.localtime()
        subst = {
            'today': '%i-%02i-%02i'%(year,month,day),
            'entry_date': self.pub_date.date().isoformat(),
            'id': self.id,
            'entry_title': re.sub(BADFNCHARS,'_',entry.get('title')),
            'feed_href': re.sub(BADFNCHARS,'_',feed.href.split('://')[-1]),
            'feed_title': re.sub(BADFNCHARS,'_',feed.feed.get('title',feed.href)),
            'original_filename': re.sub(BADFNCHARS,'_',filename),
        }
        return FILENAME.format(subst)

def process_feed(url):
    feed = feedparser.parse(url)

    if feed.bozo and isinstance(feed.bozo_exception, 
                                (type(feedparser.NonXMLContentType), type(feedparser.CharacterEncodingOverride))):
        l.error("Erroneous feed URL: %s (%s)"%(url, type(feed.bozo_exception)))
        return

    l.info("Checking feed: %s"%feed.feed.title)
    
    feed.entries.reverse()
    for entry in feed.entries:
        EntryProcessor(entry, feed)

def init():
    if not os.path.exists(args.basedir):
        l.info("Creating base dir %s"%args.basedir)
        os.makedirs(args.basedir,0700)

    subscriptions = args.basedir + os.sep + 'subscriptions'
    if not os.path.exists(subscriptions):
        l.info("Creating empty subscriptions file %s"%subscriptions)
        open(subscriptions,'a').write("# Add your RSS/ATOM subscriptions here.\n\n")

    SeenEntry._connection = sqlite.builder()(expanduser(args.basedir + '/seen.sqlite'), debug=False)
    SeenEntry.createTable(ifNotExists=True)


if __name__ == '__main__':
    init()

    for url in map(lambda x: x.strip(), open(args.basedir + os.sep + 'subscriptions')):
        if not url or url[0] in CONFIGCOMMENT:
            process_feed(url)

<commit_msg>fix some problems, when initializing app for the first time.<commit_after>#!/usr/bin/env python

import feedparser
import time
import hashlib
import os
from os.path import expanduser
import logging
import re
import cgi
import requests
import argparse
from datetime import datetime as dt
from clint.textui import progress
from sqlobject import SQLObject, sqlite, DateTimeCol, UnicodeCol

parser = argparse.ArgumentParser(description='Download podcasts via the command line.')
parser.add_argument('--no-download', action='store_true',
                   help="Don't download any files. Just mark as read.")
parser.add_argument('--podcastdir', '-p', default='~/Downloads/podcasts', 
    help="Folder to download podcast files to.")
parser.add_argument('--basedir', '-b', default='~/.upodder', 
    help="Folder to store subscriptions and seen database.")
parser.add_argument('--oldness', '-o', default=30, 
    help="Skip entries older than X days.")
parser.add_argument('--mark-seen', action='store_true',
    help="Just mark all entries as seen and exit.")
parser.add_argument('--log-level', '-l', default=5, 
    help="Logging level. 5=Debug.")
args = parser.parse_args()

YES = [1,"1","on","yes","Yes","YES","y","Y","true","True","TRUE","t","T"]
CONFIGCOMMENT = ['#',';','$',':','"',"'"]
BADFNCHARS = re.compile('[^\w]+',re.LOCALE)
TEMPDIR = '/tmp/upodder'
FILENAME = '%(entry_title)s.mp3'

# Initializing logging
l = logging.Logger('upodder', logging.DEBUG)
stderrHandler = logging.StreamHandler()
stderrHandler.setFormatter(logging.Formatter('%(message)s'))
l.addHandler(stderrHandler)

class SeenEntry(SQLObject):
    "Represents a single feed item, seen before. Used to keep track of download status."
    hashed = UnicodeCol()
    pub_date = DateTimeCol()

class EntryProcessor(object):
    "Processes single feed entry"
    def __init__(self, entry, feed):
        self.hashed = hashlib.sha1(entry['title'].encode('ascii', 'ignore')).hexdigest()
        self.pub_date = dt.fromtimestamp(time.mktime(entry.published_parsed))

        if args.mark_seen:
            SeenEntry(pub_date=self.pub_date, hashed=self.hashed)
            l.debug("Marking as seen: %s"%(entry['title']))
            return

        # Let's check if we worked on this entry earlier...
        if SeenEntry.select(SeenEntry.q.hashed == self.hashed).count() > 0:
            l.debug("Already seen: %s"%(entry['title']))
            return
        
        # Let's check the entry's date
        if (dt.now() - self.pub_date).days > args.oldness:
            l.debug("Too old for us: %s"%entry['title'])
            return

        # Search for mpeg enclosures
        for enclosure in filter(lambda x: x.get('type') == 'audio/mpeg',entry.get('enclosures',[])):
            # Work only with first found audio/mpeg enclosure (Bad Thing? maybe :( )
            if self._download_enclosure(enclosure, entry, feed, args.no_download):
                SeenEntry( pub_date=self.pub_date, hashed=self.hashed)
            break

    def _download_enclosure(self, enclosure, entry, feed, no_download=False):
        """Performs downloading of specified file. Returns True on success and False in other case"""

        downloadto = TEMPDIR + os.sep + self.hashed

        if no_download:
            l.debug("Would download %s"%enclosure['href'])
            return True

        try:
            """Downloads URL to file, returns file name of download (from URL or Content-Disposition)"""
            if not os.path.exists(os.path.dirname(downloadto)):
                os.makedirs(os.path.dirname(downloadto),0700)

            l.debug("Downloading %s"%enclosure['href'])
            r = requests.get(enclosure['href'], stream=True)
            with open(downloadto, 'wb') as f:
                if 'content-length' in r.headers:
                    total_length = int(r.headers['content-length'])
                    # print total_length, enclosure['length'] #TODO: which measure is more reliable?
                    r_iter = progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1)
                else:
                    r_iter = r.iter_content(chunk_size=1024)
                for chunk in r_iter: 
                    if chunk:
                        f.write(chunk)
                        f.flush()

            filename = cgi.parse_header(r.headers.get('content-disposition'))[1]['filename']
            if not filename:
                filename = "Untitled.mp3"

        except KeyboardInterrupt:
            l.info("Download aborted by Ctrl+c")
            try:
                user_wish = raw_input("Do you like to mark item as read? (y/N): ")
                if user_wish in YES:
                    return True
            except KeyboardInterrupt:
                print "No"
            return False

        # Move downloaded file to its final destination
        moveto = args.podcastsdir + os.sep + self._generate_filename(filename, entry, feed)
        l.debug("Moving {%s} to {%s}"%(downloadto,moveto))
        if not os.path.exists(os.path.dirname(moveto)): os.makedirs(os.path.dirname(moveto),0750)
        os.rename(downloadto,moveto)
        return True

    def _generate_filename(self, filename, entry, feed):
        """Generates file name for this enclosure based on config settins"""
        (year,month,day,hour,minute,second,weekday,yearday,leap) = time.localtime()
        subst = {
            'today': '%i-%02i-%02i'%(year,month,day),
            'entry_date': self.pub_date.date().isoformat(),
            'id': self.id,
            'entry_title': re.sub(BADFNCHARS,'_',entry.get('title')),
            'feed_href': re.sub(BADFNCHARS,'_',feed.href.split('://')[-1]),
            'feed_title': re.sub(BADFNCHARS,'_',feed.feed.get('title',feed.href)),
            'original_filename': re.sub(BADFNCHARS,'_',filename),
        }
        return FILENAME.format(subst)

def process_feed(url):
    feed = feedparser.parse(url)

    if feed.bozo and isinstance(feed.bozo_exception, 
                                (type(feedparser.NonXMLContentType), type(feedparser.CharacterEncodingOverride))):
        l.error("Erroneous feed URL: %s (%s)"%(url, type(feed.bozo_exception)))
        return

    l.info("Checking feed: %s"%feed.feed.title)
    
    feed.entries.reverse()
    for entry in feed.entries:
        EntryProcessor(entry, feed)

def init():
    if not os.path.exists(expanduser(args.basedir)):
        l.info("Creating base dir %s"%args.basedir)
        os.makedirs(expanduser(args.basedir), 0700)

    subscriptions = expanduser(args.basedir) + os.sep + 'subscriptions'
    if not os.path.exists(subscriptions):
        l.info("Creating empty subscriptions file %s"%subscriptions)
        open(subscriptions,'a').write("# Add your RSS/ATOM subscriptions here.\n\n")

    SeenEntry._connection = sqlite.builder()(expanduser(args.basedir + os.sep + 'seen.sqlite'), debug=False)
    SeenEntry.createTable(ifNotExists=True)


if __name__ == '__main__':
    init()

    for url in map(lambda x: x.strip(), open(expanduser(args.basedir) + os.sep + 'subscriptions')):
        if url and url[0] not in CONFIGCOMMENT:
            process_feed(url)
    
    l.info('Done updating feeds.')


{'fraction_non_alphanumeric': 0.1362198636395586, 'fraction_numerical': 0.0059042665354607435, 'mean_word_length': 2.757063638764193, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 51, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3037935', 'n_tokens_mistral': 4374, 'n_tokens_neox': 4176, 'n_words': 994}