<commit_before>module BibSync
  module Actions
    class FetchFromArXiv
      SliceSize = 20

      include Log
      include Utils

      def initialize(options)
        raise 'Fetch must be set' unless @fetch = options[:fetch]
        raise 'Directory must be set' unless @dir = options[:dir]
      end

      def run
        ids = []
        urls = []

        @fetch.each do |url|
          if url =~ %r{^http://arxiv.org/abs/(\d+\.\d+)$}
            ids << $1
          else
            urls << url
          end
        end

        unless urls.empty?
          notice 'Starting browser for non-arXiv urls'
          urls.each do |url|
            info "Opening #{url}"
            `xdg-open #{Shellwords.escape url}`
          end
        end

        unless ids.empty?
          notice 'Downloading from arXiv'
          ids.each_slice(SliceSize) do |ids|
            begin
              xml = fetch_xml("http://export.arxiv.org/api/query?id_list=#{ids.join(',')}&max_results=#{SliceSize}")
              xml.xpath('//entry/id').map(&:content).each_with_index do |id, i|
                id.gsub!('http://arxiv.org/abs/', '')
                info 'arXiv download', :key => id
                arxiv_download(@dir, id)
              end
            rescue => ex
              error('arXiv query failed', :ex => ex)
            end
          end
        end
      end
    end
  end
end
<commit_msg>downloader accepts arxiv ids<commit_after>module BibSync
  module Actions
    class FetchFromArXiv
      SliceSize = 20

      include Log
      include Utils

      def initialize(options)
        raise 'Fetch must be set' unless @fetch = options[:fetch]
        raise 'Directory must be set' unless @dir = options[:dir]
      end

      def run
        arxivs = []
        urls = []

        @fetch.each do |url|
          if url =~ /\A(\d+\.\d+)(v\d+)?\Z/
            arxivs << $1
          if url =~ %r{\Ahttp://arxiv.org/abs/(\d+\.\d+)\Z}
            arxivs << $1
          else
            urls << url
          end
        end

        unless urls.empty?
          notice 'Starting browser for non-arXiv urls'
          urls.each do |url|
            info "Opening #{url}"
            `xdg-open #{Shellwords.escape url}`
          end
        end

        unless arxivs.empty?
          notice 'Downloading from arXiv'
          arxivs.each_slice(SliceSize) do |ids|
            begin
              xml = fetch_xml("http://export.arxiv.org/api/query?id_list=#{ids.join(',')}&max_results=#{SliceSize}")
              xml.xpath('//entry/id').map(&:content).each_with_index do |id, i|
                id.gsub!('http://arxiv.org/abs/', '')
                info 'arXiv download', :key => id
                arxiv_download(@dir, id)
              end
            rescue => ex
              error('arXiv query failed', :ex => ex)
            end
          end
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12129565816678153, 'fraction_numerical': 0.002412129565816678, 'mean_word_length': 1.431323283082077, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5886277', 'n_tokens_mistral': 911, 'n_tokens_neox': 841, 'n_words': 220}