<commit_before>require 'feed-normalizer'
class BlogRoll < ActiveRecord::Base
  self.table_name = 'public.blog_rolls'
  has_many :blog_entries

  def self.harvest_new_entries()
    rollin = BlogRoll.where('active_flag = true')
    rollin.each do |r|
      parse_feed_normal(r[:feed_uri], r[:id], Time.now - 3.days)
    end
  end

  def self.create(title, author, uri, feed_uri, feed_type)
      roll = BlogRoll.new
      roll.title = title
      roll.author = author
      roll.blog_uri = uri
      roll.feed_uri = feed_uri
      roll.feed_type = feed_type
      roll.save
      
      #back fill the last 3 months
      parse_feed_normal(roll.feed_uri, roll.id, Time.now - 3.months)
  end



  def self.parse_feed_normal(feed_uri, blog_id, datefilter)
    
    # this method is still pretty gross and needs help
    counter = 2
    begin
        p counter
        feed = FeedNormalizer::FeedNormalizer.parse open(feed_uri)
        #feed = FeedNormalizer::FeedNormalizer.parse open("http://datachomp.com/atom.xml")
        if feed.entries.nil?
          sleep 2
          p 'the lion sleeps tonight'
          feed = FeedNormalizer::FeedNormalizer.parse open(feed_uri)
        end
      rescue Timeout::Error => e
        p 'timeouts'  
        p e.message
        return false
      rescue OpenURI::HTTPError => e
        p 'this whole thing is broken'
        p e.message
        return false
      rescue NoMethodError => e
        p 'no method here'
        p e.message
        return false
      rescue RuntimeError  => e
        p 'it broke'

        p e.message
        (counter-=1) < 0 ? raise : retry  #lets just rety twice
      end
        
    feed.entries.each do |item|
      dateland = item.date_published  || item.last_updated

      if dateland.to_date > datefilter
        
        toppy = BlogEntry.where("entry_uri = ?", item.id).select(:id).first
        if toppy.nil?
          entry = BlogEntry.new
          entry.blog_roll_id = blog_id
          entry.title = item.title
          entry.entry_uri = item.id
          entry.posted_date = dateland
          entry.save
        end
      end  #end if
    end
  end

end<commit_msg>use google instead of the old feed parser. thanks jeremy<commit_after>require "googleajax"
class BlogRoll < ActiveRecord::Base
  self.table_name = 'public.blog_rolls'
  has_many :blog_entries

  def self.harvest_new_entries()
    rollin = BlogRoll.where('active_flag = true')
    rollin.each do |r|
      parse_feed_normal(r[:feed_uri], r[:id], Time.now - 3.days)
    end
  end

  def self.create(title, author, uri, feed_uri, feed_type)
      roll = BlogRoll.new
      roll.title = title
      roll.author = author
      roll.blog_uri = uri
      roll.feed_uri = feed_uri
      roll.feed_type = feed_type
      roll.save

      #back fill the last 3 months
      parse_feed_normal(roll.feed_uri, roll.id, Time.now - 3.months)
  end


  def self.parse_feed_normal(feed_uri, blog_id, datefilter)
    GoogleAjax.referrer = "techlahoma.com"

    begin
        feedinfo = GoogleAjax::Feed.load(feed_uri)
        feedinfo[:entries].each do |c|

          if c[:published_date].to_date > datefilter
            toppy = BlogEntry.where("entry_uri = ?", c[:link]).select(:id).first

            if toppy.nil?
              entry = BlogEntry.new
              entry.blog_roll_id = blog_id
              entry.title = c[:title]
              entry.entry_uri = c[:link]
              entry.posted_date = c[:published_date]
              entry.save
            end
          end#end date check
        end #end feed info loop
    end
  end

end
=begin
BlogRoll.create('<NAME>', '<NAME>', 'http://rousseau.io/', 'http://rousseau.io/atom.xml', 'atom' )
BlogRoll.create('Groove Coder', '<NAME>', 'http://groovecoder.com/', 'http://groovecoder.com/feed/', 'feed' )
Lu<NAME> - http://groovecoder.com/feed/
BlogRoll.create('<NAME>', '<NAME>', 'http://buddylindsey.com/', 'http://buddylindsey.com/feed/', 'live' )


BlogRoll.create('<NAME>', '<NAME>', 'http://devin-clark.com/', 'http://devin-clark.com/rss/', 'rss' )
BlogRoll.create('<NAME>', '<NAME>', 'http://www.simiansblog.com/', 'http://www.simiansblog.com/rss?containerid=52', 'word' )
http://devin-clark.com/rss/
=end
{'fraction_non_alphanumeric': 0.11977186311787072, 'fraction_numerical': 0.0028517110266159697, 'mean_word_length': 2.353784860557769, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6851158', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1371, 'n_words': 307}