<commit_before>import feedparser
import threading
from models.episode import Episode
from copy import deepcopy
import log
import os

class EpisodeWorker(threading.Thread):

  def __init__(self, storer, series, i):
    """
    Constructor for thread that will request the RSS of a
    particular podcast series, parse the series details
    and episode information, and save the information
    w/`storer`
    """
    super(EpisodeWorker, self).__init__()
    self.logger = log.logger
    self.storer = storer
    self.series = series # All series
    self.i      = i

  def request_rss(self, url):
    """
    Uses information in `line` to request and return the
    RSS feed
    """
    return feedparser.parse(url)

  def run(self):
    """
    Run the task - compose full series + add to our results
    """
    while self.i < len(self.series):
      # Grab line + RSS
      s = self.series[self.i]
      rss = self.request_rss(s.feedUrl)

      # Compose Episodes
      ep_dicts = []
      for entry in rss['entries']:
        ep_dicts.append(Episode(s, entry).__dict__)

      # Build result dict
      result_dict = dict()
      result_dict['series'] = deepcopy(s.__dict__)
      result_dict['series']['genres'] = \
        result_dict['series']['genres'].split(';')
      result_dict['series']['type'] = 'series'
      result_dict['episodes'] = ep_dicts

      # Store podcast
      self.storer.store(result_dict)

      # Move onto the next one
      self.i += 20
      self.logger.info('Retrieved {}'.format(str(s.id)))<commit_msg>log clarity<commit_after>import feedparser
import threading
from models.episode import Episode
from copy import deepcopy
import log
import os
import pdb

class EpisodeWorker(threading.Thread):

  def __init__(self, storer, series, i):
    """
    Constructor for thread that will request the RSS of a
    particular podcast series, parse the series details
    and episode information, and save the information
    w/`storer`
    """
    super(EpisodeWorker, self).__init__()
    self.logger = log.logger
    self.storer = storer
    self.series = series # All series
    self.i      = i

  def request_rss(self, url):
    """
    Uses information in `line` to request and return the
    RSS feed
    """
    return feedparser.parse(url)

  def run(self):
    """
    Run the task - compose full series + add to our results
    """
    while self.i < len(self.series):
      # Grab line + RSS
      s = self.series[self.i]
      rss = self.request_rss(s.feedUrl)

      # Compose Episodes
      ep_dicts = []
      for entry in rss['entries']:
        ep_dicts.append(Episode(s, entry).__dict__)

      # Build result dict
      result_dict = dict()
      result_dict['series'] = deepcopy(s.__dict__)
      result_dict['series']['genres'] = \
        result_dict['series']['genres'].split(';')
      result_dict['series']['type'] = 'series'
      result_dict['episodes'] = ep_dicts

      # Store podcast
      self.storer.store(result_dict)

      # Move onto the next one
      self.i += 20
      self.logger.info('Retrieved and stored {}'.format(str(s.id)))

if __name__=="__main__":
  worker = EpisodeWorker(None, None, 0)
  rss = worker.request_rss("http://feeds.soundcloud.com/users/soundcloud:users:154009125/sounds.rss")
  pdb.set_trace()
{'fraction_non_alphanumeric': 0.1162507608034084, 'fraction_numerical': 0.00426049908703591, 'mean_word_length': 2.867058823529412, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1522572', 'n_tokens_mistral': 1044, 'n_tokens_neox': 1004, 'n_words': 277}