<commit_before>class FeedUpdaterWorker
  @queue = :feed_updating

  class << self
    def perform(stream_id)
      stream = Stream.find(stream_id)
      feed = stream.feed
      parsed_feed = parse_feed!(feed, stream)
      items = parsed_feed.entries

      items.each do |rss_item|

        if feed.new_item?(rss_item)
          Rails.logger.debug "New item #{rss_item.title} for feed #{feed.name}, id: #{feed.id}"

          item = feed.items.create_from_rss(rss_item, parsed_feed) 

          enqueue(stream, item)

        elsif 
          Rails.logger.debug "Item #{rss_item.title} already exists for feed #{feed.name}, id: #{feed.id}"
        end
      end
    end

    private
    
    def enqueue(stream, item)
      if category_matches?(stream, item)
        # TODO: only grab active twitter accounts
        stream.twitter_accounts.each do |account|
          Resque.enqueue(TwitterUpdaterWorker, item.id, account.id)
        end

        stream.email_lists.each do |list|
          Resque.enqueue(EmailWorker, item.id, list.id)
        end
      end
    end

    def parse_feed!(feed, stream)
      parsed_feed = Feedzirra::Feed.fetch_and_parse(feed.url)

      if !parsed_feed || parsed_feed == 0
        stream.update_attributes(:active => false)
        raise UnparsableFeedError.new
      end

      parsed_feed
    end

    def category_matches?(stream, item)
      result = true # if the stream doesn't define categories to include, assume all categories

      if stream.included_categories.present?
        stream_categories = stream.included_categories.split(",").collect { |str| str.strip }
        item_categories = item.categories

        item_categories.each do |item_category|
          if stream_categories.include?(item_category)
            result = true
            break;
          end
        result = false
        end
      end
      result
    end

  end

  class UnparsableFeedError < StandardError; end
end
<commit_msg>hook up facebook updater to feed updater<commit_after>class FeedUpdaterWorker
  @queue = :feed_updating

  class << self
    def perform(stream_id)
      stream = Stream.find(stream_id)
      feed = stream.feed
      parsed_feed = parse_feed!(feed, stream)
      items = parsed_feed.entries

      items.each do |rss_item|

        if feed.new_item?(rss_item)
          Rails.logger.debug "New item #{rss_item.title} for feed #{feed.name}, id: #{feed.id}"

          item = feed.items.create_from_rss(rss_item, parsed_feed) 

          enqueue(stream, item)

        elsif 
          Rails.logger.debug "Item #{rss_item.title} already exists for feed #{feed.name}, id: #{feed.id}"
        end
      end
    end

    private
    
    def enqueue(stream, item)
      if category_matches?(stream, item)
        # TODO: only grab active twitter accounts
        stream.twitter_accounts.each do |account|
          Resque.enqueue(TwitterUpdaterWorker, item.id, account.id)
        end

        stream.email_lists.each do |list|
          Resque.enqueue(EmailWorker, item.id, list.id)
        end

        stream.facebook_accounts.each do |account|
          Resque.enqueue(FacebookUpdaterWorker, item.id, account.id)
        end
      end
    end

    def parse_feed!(feed, stream)
      parsed_feed = Feedzirra::Feed.fetch_and_parse(feed.url)

      if !parsed_feed || parsed_feed == 0
        stream.update_attributes(:active => false)
        raise UnparsableFeedError.new
      end

      parsed_feed
    end

    def category_matches?(stream, item)
      result = true # if the stream doesn't define categories to include, assume all categories

      if stream.included_categories.present?
        stream_categories = stream.included_categories.split(",").collect { |str| str.strip }
        item_categories = item.categories

        item_categories.each do |item_category|
          if stream_categories.include?(item_category)
            result = true
            break;
          end
        result = false
        end
      end
      result
    end

  end

  class UnparsableFeedError < StandardError; end
end

{'fraction_non_alphanumeric': 0.08517427589592538, 'fraction_numerical': 0.0004909180166912126, 'mean_word_length': 2.334697217675941, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4649378', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1179, 'n_words': 293}