<commit_before>require 'arxivsync'

namespace :arxiv do
  desc "Update database with recent papers"
  task oai_update: :environment do
    time = Time.now.utc

    last_paper = Paper.order("submit_date desc").first

    if last_paper.nil?
      date = Time.now-1.days
    else
      date = last_paper.submit_date
    end

    # Do this in a single transaction to avoid any database consistency issues
    bulk_papers = []
    ArxivSync.get_metadata(from: date.to_date) do |resp, papers|
      bulk_papers += papers
    end<commit_msg>Sync from last pubdate rather than submit_date<commit_after>require 'arxivsync'

namespace :arxiv do
  desc "Update database with recent papers"
  task oai_update: :environment do
    time = Time.now.utc

    last_paper = Paper.order("submit_date desc").first

    if last_paper.nil?
      fromdate = Time.now-1.days
    else
      fromdate = last_paper.pubdate
    end

    # Do this in a single transaction to avoid any database consistency issues
    bulk_papers = []
    ArxivSync.get_metadata(from: fromdate.to_date) do |resp, papers|
      bulk_papers += papers
    end
{'fraction_non_alphanumeric': 0.0757439134355275, 'fraction_numerical': 0.0018034265103697023, 'mean_word_length': 3.188679245283019, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4988099', 'n_tokens_mistral': 350, 'n_tokens_neox': 333, 'n_words': 103}