<commit_before>module NDC9
  def self.last_request
    $redis.get "last_request"
  end

  def self.fetch(isbn, opt={})
    cache = (opt["cache"] || true)
    self.cache(isbn, {:cache=>cache}) do
        unless self.last_request.nil?
          time = (Time.now - self.last_request)
          if  tiem < 1.0 then
            sleep (1.0-time)
          end
        end

        source = open("http://iss.ndl.go.jp/api/opensearch?dpid=iss-ndl-opac&isbn=#{isbn}&mediatype=1").read
        doc    = Nokogiri::XML.parse(source)
        doc.xpath("//channel/item/dc:subject[@xsi:type='dcndl:NDC9']/text()").to_s 
    end
  end

  def self.multi_fetch(isbn_list, opt={})
    raise ArgumentError, "isbn_list should be Array" unless isbn_list.instance_of? Array
    isbn_list.map do |isbn|
      [isbn, self.fetch(isbn, opt)]
    end
  end

  def self.bulk_request(isbn_list, opt={})
    raise ArgumentError, "isbn_list should be Array" unless isbn_list.instance_of? Array
    request_id = Digest::SHA1.hexdigest(isbn_list.to_s)

    $redis.multi do
      isbn_list.each do |isbn|
        $redis.lpush "request:#{request_id}", isbn
      end
    end
    request_id
  end

  def self.bulk_request_exists?(request_id)
    $redis.exists("request:#{request_id}") or $redis.exists("result:#{request_id}")
  end

  def self.bulk_fetch(request_id)
    key = "request:#{request_id}"
    raise unless $redis.exists key

    while $redis.llen(key) > 0 do
      isbn = $redis.lindex(key, 0)
      ndc9 = self.fetch(isbn)
      $redis.multi do
        $redis.lpop key
        $redis.hset "result:#{request_id}", isbn, ndc9
      end
    end

    $redis.expire "result:#{request_id}", 43200 # 6 hours
    $redis.del "request:#{request_id}"

    true<commit_msg>refactoring<commit_after>module NDC9
  BULK_EXPIRE = 43200 # 6 hours

  def self.last_request
    $redis.get "last_request"
  end

  def self.fetch(isbn, opt={})
    cache = (opt["cache"] || true)
    self.cache(isbn, {:cache=>cache}) do
        unless self.last_request.nil?
          time = (Time.now - self.last_request)
          if  tiem < 1.0 then
            sleep (1.0-time)
          end
        end

        source = open("http://iss.ndl.go.jp/api/opensearch?dpid=iss-ndl-opac&isbn=#{isbn}&mediatype=1").read
        doc    = Nokogiri::XML.parse(source)
        doc.xpath("//channel/item/dc:subject[@xsi:type='dcndl:NDC9']/text()").to_s 
    end
  end

  def self.multi_fetch(isbn_list, opt={})
    raise ArgumentError, "isbn_list should be Array" unless isbn_list.instance_of? Array
    isbn_list.map do |isbn|
      [isbn, self.fetch(isbn, opt)]
    end
  end

  def self.bulk_request(isbn_list, opt={})
    raise ArgumentError, "isbn_list should be Array" unless isbn_list.instance_of? Array
    request_id = Digest::SHA1.hexdigest(isbn_list.to_s)

    $redis.multi do
      isbn_list.each do |isbn|
        $redis.lpush "request:#{request_id}", isbn
      end
    end
    request_id
  end

  def self.bulk_request_exists?(request_id)
    $redis.exists("request:#{request_id}") or $redis.exists("result:#{request_id}")
  end

  def self.bulk_fetch(request_id)
    key = "request:#{request_id}"
    raise unless $redis.exists key

    while $redis.llen(key) > 0 do
      isbn = $redis.lindex(key, 0)
      ndc9 = self.fetch(isbn)
      $redis.multi do
        $redis.lpop key
        $redis.hset "result:#{request_id}", isbn, ndc9
      end
    end

    $redis.expire "result:#{request_id}", BULK_EXPIRE
    $redis.del "request:#{request_id}"

    true
{'fraction_non_alphanumeric': 0.1368180519851471, 'fraction_numerical': 0.010282776349614395, 'mean_word_length': 2.8273224043715848, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1112460', 'n_tokens_mistral': 1349, 'n_tokens_neox': 1311, 'n_words': 247}