<commit_before>require 'thread'

# Simple class to retrieve, cache and filter results of RSS feeds using Feedjira
class RSSCache
  
  DEFAULT_OPTIONS = {
    :timeout_secs => 60*60,
    :poll_secs => 60,
    :filter => lambda {|entries| return entries}
  }
  
  def initialize url, options = {}
    
    @url = url
    @last_updated = nil
    @entries = nil
    @lock = Mutex.new
    
    @options = DEFAULT_OPTIONS.merge(options)
    
    Thread.new do
      begin
        while true do
          self.update false
          sleep @options[:poll_secs]
        end
      rescue Exception => ex
        backtrace = ex.backtrace.join("\n")
        puts "#{Time.now.to_formatted_s :db} :: Exception thrown in RSSCache Thread loop :: #{ex.message}\n#{backtrace}"
      end
    end
  end
  
  def update jit_update
    @lock.synchronize do
      if @last_updated.nil? or @last_updated < Time.now - @options[:timeout_secs]
        puts "#{Time.now.to_formatted_s :db} :: RSSCache Update #{jit_update ? '(JIT) ' : ''}:: #{@url} :: Will cache for #{@options[:timeout_secs]} seconds\n"
        raw_entries = Feedjira::Feed.fetch_and_parse(@url).entries
        
        @entries = @options[:filter].call raw_entries unless raw_entries.size == 0
        @last_updated = Time.now unless @entries.size == 0
      end
    end
  end
  
  def get 
    self.update true
    @entries
  end
  
end<commit_msg>Anticipate and handle Feedjira parse failures.<commit_after>require 'thread'
require 'feedjira'
require 'active_support/core_ext/string/inflections'

# Simple class to retrieve, cache and filter results of RSS feeds using Feedjira
class RSSCache
  
  DEFAULT_OPTIONS = {
    :timeout_secs => 60*60,
    :poll_secs => 60,
    :filter => lambda {|entries| return entries}
  }
  
  def initialize url, options = {}
    
    @url = url
    @last_updated = nil
    @entries = nil
    @lock = Mutex.new
    
    @options = DEFAULT_OPTIONS.merge(options)
    
    Thread.new do
      begin
        while true do
          self.update false
          sleep @options[:poll_secs]
        end
      rescue Exception => ex
        backtrace = ex.backtrace.join("\n")
        puts "#{Time.now.to_formatted_s :db} :: Exception thrown in RSSCache Thread loop :: #{ex.message}\n#{backtrace}"
      end
    end
  end
  
  def update jit_update
    @lock.synchronize do
      if @last_updated.nil? or @last_updated < Time.now - @options[:timeout_secs]
        puts "#{Time.now.to_formatted_s :db} :: RSSCache Update #{jit_update ? '(JIT) ' : ''}:: #{@url} :: Fetching RSS\n"
        feed_result = Feedjira::Feed.fetch_and_parse(@url)
        # Belt and braces check that we've actually been given back a Feedjira Parser class and not a Fixnum 
        # or perhaps something else entirely, depending on what Feedjira felt like returning (seems to return
        # HTTP status code as a Fixnum when parsing fails)
        if not feed_result.nil? and Feedjira::Parser.constants.include? feed_result.class.name.demodulize.to_sym
          raw_entries = feed_result.entries
          @entries = @options[:filter].call raw_entries unless raw_entries.size == 0
          @last_updated = Time.now unless @entries.size == 0
          puts "#{Time.now.to_formatted_s :db} :: RSSCache Update #{jit_update ? '(JIT) ' : ''}:: #{@url} :: Success - will cache for #{@options[:timeout_secs]} seconds\n"
        else
          puts "#{Time.now.to_formatted_s :db} :: RSSCache Update #{jit_update ? '(JIT) ' : ''}:: #{@url} :: Fail - result was #{feed_result.nil? ? 'nil' : feed_result}\n"
        end
      end
    end
  end
  
  def get 
    self.update true
    @entries
  end
  
end
{'fraction_non_alphanumeric': 0.11312465678198792, 'fraction_numerical': 0.004393190554640308, 'mean_word_length': 2.6466466466466465, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '445103', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1163, 'n_words': 319}