<commit_before>
require 'miga/remote_dataset/base'

class MiGA::RemoteDataset
  include MiGA::RemoteDataset::Base

  # Class-level
  class << self
    ##
    # Download data from the +universe+ in the database +db+ with IDs +ids+ and
    # in +format+. If passed, it saves the result in +file+. Additional
    # parameters specific to the download method can be passed using +extra+.
    # Returns String.
    def download(universe, db, ids, format, file = nil, extra = [])
      ids = [ids] unless ids.is_a? Array
      case @@UNIVERSE[universe][:method]
      when :rest
        doc = download_rest(universe, db, ids, format, extra)
      when :net
        doc = download_net(universe, db, ids, format, extra)
      end
      unless file.nil?
        ofh = File.open(file, 'w')
        ofh.print doc
        ofh.close
      end
      doc
    end

    ##
    # Download data using a REST method from the +universe+ in the database +db+
    # with IDs +ids+ and in +format+. Additional URL parameters can be passed
    # using +extra+. Returns the doc as String.
    def download_rest(universe, db, ids, format, extra = [])
      u = @@UNIVERSE[universe]
      url = sprintf(u[:url], db, ids.join(","), format, *extra)
      download_url url
    end

    ##
    # Download data using a GET request from the +universe+ in the database +db+
    # with IDs +ids+ and in +format+. Additional URL parameters can be passed
    # using +extra+. Returns the doc as String.
    alias download_net download_rest

    ##
    # Download the given +url+ and return the result regardless of response
    # code. Attempts download up to three times before raising Net::ReadTimeout.
    def download_url(url)
      doc = ''
      @timeout_try = 0
      begin
        open(url, open_timeout: 600, read_timeout: 600) { |f| doc = f.read }
      rescue => e
        @timeout_try += 1
        raise e if @timeout_try >= 3
        retry
      end
      doc
    end

    ##
    # Looks for the entry +id+ in +dbfrom+, and returns the linked
    # identifier in +db+ (or nil).
    def ncbi_map(id, dbfrom, db)
      doc = download(:ncbi_map, dbfrom, id, :json, nil, [db])
      return if doc.empty?
      tree = JSON.parse(doc, symbolize_names: true)
      [:linksets, 0, :linksetdbs, 0, :links, 0].each do |i|
        tree = tree[i]
        break if tree.nil?
      end
      tree<commit_msg>Older Ruby didn't support open_timeout in open-uri<commit_after>
require 'miga/remote_dataset/base'

class MiGA::RemoteDataset
  include MiGA::RemoteDataset::Base

  # Class-level
  class << self
    ##
    # Download data from the +universe+ in the database +db+ with IDs +ids+ and
    # in +format+. If passed, it saves the result in +file+. Additional
    # parameters specific to the download method can be passed using +extra+.
    # Returns String.
    def download(universe, db, ids, format, file = nil, extra = [])
      ids = [ids] unless ids.is_a? Array
      case @@UNIVERSE[universe][:method]
      when :rest
        doc = download_rest(universe, db, ids, format, extra)
      when :net
        doc = download_net(universe, db, ids, format, extra)
      end
      unless file.nil?
        ofh = File.open(file, 'w')
        ofh.print doc
        ofh.close
      end
      doc
    end

    ##
    # Download data using a REST method from the +universe+ in the database +db+
    # with IDs +ids+ and in +format+. Additional URL parameters can be passed
    # using +extra+. Returns the doc as String.
    def download_rest(universe, db, ids, format, extra = [])
      u = @@UNIVERSE[universe]
      url = sprintf(u[:url], db, ids.join(","), format, *extra)
      download_url url
    end

    ##
    # Download data using a GET request from the +universe+ in the database +db+
    # with IDs +ids+ and in +format+. Additional URL parameters can be passed
    # using +extra+. Returns the doc as String.
    alias download_net download_rest

    ##
    # Download the given +url+ and return the result regardless of response
    # code. Attempts download up to three times before raising Net::ReadTimeout.
    def download_url(url)
      doc = ''
      @timeout_try = 0
      begin
        open(url, read_timeout: 600) { |f| doc = f.read }
      rescue => e
        @timeout_try += 1
        raise e if @timeout_try >= 3
        retry
      end
      doc
    end

    ##
    # Looks for the entry +id+ in +dbfrom+, and returns the linked
    # identifier in +db+ (or nil).
    def ncbi_map(id, dbfrom, db)
      doc = download(:ncbi_map, dbfrom, id, :json, nil, [db])
      return if doc.empty?
      tree = JSON.parse(doc, symbolize_names: true)
      [:linksets, 0, :linksetdbs, 0, :links, 0].each do |i|
        tree = tree[i]
        break if tree.nil?
      end
      tree
{'fraction_non_alphanumeric': 0.10513036164844407, 'fraction_numerical': 0.004415475189234651, 'mean_word_length': 2.447101449275362, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2997206', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1460, 'n_words': 545}