<commit_before>require 'open-uri'

module Feed::Parser

  def update!
    existing = self.articles.all

    entries.each do |entry|
      if ntry = existing.detect{|e| e.entry_id == entry.entry_id}
        ntry.attributes = entry.attributes
        ntry.save!
      else
        entry.save!(source: self.slug, user: self.user)
      end
    end

    excess = self.articles.desc(:date).skip(10)
    Article.collection.remove( "_id" => {"$in" => excess.map(&:id)} )
  end

  def parse
    @xml = 
      begin
        xml = open(url, 'r', read_timeout: 5.0)
        RSS::Parser.parse(xml)
      rescue
        $stderr.puts "There was an error fetching the feed #{name}, for #{user.try(:to_label) || 'no user'}"
        nil
      end
  end

  def xml
    @xml || parse && @xml
  end

  def entries
    return [] unless xml.present?

    if xml.respond_to?(:entries)<commit_msg>More logs in the worker<commit_after>require 'open-uri'

module Feed::Parser

  def update!
    existing = self.articles.all

    entries.each do |entry|
      if ntry = existing.detect{|e| e.entry_id == entry.entry_id}
        ntry.attributes = entry.attributes
        ntry.save!
      else
        entry.save!(source: self.slug, user: self.user)
      end
    end

    excess = self.articles.desc(:date).skip(10)
    Article.collection.remove( "_id" => {"$in" => excess.map(&:id)} )
  end

  def parse
    @xml = 
      begin
        xml = open(url, 'r', read_timeout: 5.0)
        RSS::Parser.parse(xml)
      rescue Exception => e
        $stderr.puts "**************************************************"
        $stderr.puts "There was an error fetching the feed #{name}, for #{user.try(:to_label) || 'no user'}"
        $stderr.puts e.message
        $stderr.puts e.backtrace
        $stderr.puts "**************************************************"
        nil
      end
  end

  def xml
    @xml || parse && @xml
  end

  def entries
    return [] unless xml.present?

    if xml.respond_to?(:entries)
{'fraction_non_alphanumeric': 0.17750882501260717, 'fraction_numerical': 0.004034291477559254, 'mean_word_length': 2.4325259515570936, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5773018', 'n_tokens_mistral': 653, 'n_tokens_neox': 627, 'n_words': 157}