<commit_before>namespace :sources do
  Rake::Task[:environment].invoke
  # How old should a source be before it needs to be refreshed?
  SOURCE_STALE = Time.now - 1.hour

  # How old should an update be before it needs to be deleted?
  UPDATE_STALE = Time.now - 2.weeks

  desc 'Get updates from our sources'
  task :update do
    sources = Source.find(:all, :order => "imported_at",
      :conditions => ['reimport = ? and imported_at < ?', true, SOURCE_STALE])
    puts "Nothing to do" unless sources.size > 0

    sources.each do |source|
      puts "Polling #{source.name}"
      begin
        events = source.create_events!
        msg = "Got #{events.size} abstract events"
      rescue Exception => e
        msg = e.to_s
      end
      source.updates.create(:status => msg)<commit_msg>Reworked sources:update task with clearer diagnostic output.<commit_after>namespace :sources do
  Rake::Task[:environment].invoke
  # How old should a source be before it needs to be refreshed?
  SOURCE_STALE = Time.now - 1.hour

  # How old should an update be before it needs to be deleted?
  UPDATE_STALE = Time.now - 2.weeks

  desc 'Get updates from our sources'
  task :update do
    sources = Source.find(:all, :order => "imported_at",
      :conditions => ['reimport = ? and imported_at < ?', true, SOURCE_STALE])
    puts "Nothing to do" unless sources.size > 0

    sources.each do |source|
      print "Polling #{source.name}: "
      begin
        events = source.create_events!
        msg = "created #{events.size} events"
      rescue Exception => e
        msg = e.to_s
      end
      puts msg
      source.updates.create(:status => msg)
{'fraction_non_alphanumeric': 0.10376213592233009, 'fraction_numerical': 0.0036407766990291263, 'mean_word_length': 2.8708920187793425, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269321', 'n_tokens_mistral': 510, 'n_tokens_neox': 489, 'n_words': 170}