<commit_before>class AuthToken < ActiveRecord::Base
  def self.client
    if @auth_token && @auth_token.high_rate_limit?
      return @auth_token.github_client
    end
    auth_token = limit(500).sample
    if auth_token.high_rate_limit?
      @auth_token = auth_token
      return auth_token.github_client
    end
    client
  end

  def self.token
    client.access_token
  end

  def self.create_multiple(array_of_tokens)
    array_of_tokens.each do |token|
      self.find_or_create_by(token: token)
    end
  end

  def high_rate_limit?
    github_client.rate_limit.remaining > 500
  rescue Octokit::Unauthorized
    false
  end<commit_msg>randomize the authtoken loading<commit_after>class AuthToken < ActiveRecord::Base
  def self.client
    if @auth_token && @auth_token.high_rate_limit?
      return @auth_token.github_client
    end
    auth_token = order("RANDOM()").limit(100).sample
    if auth_token.high_rate_limit?
      @auth_token = auth_token
      return auth_token.github_client
    end
    client
  end

  def self.token
    client.access_token
  end

  def self.create_multiple(array_of_tokens)
    array_of_tokens.each do |token|
      self.find_or_create_by(token: token)
    end
  end

  def high_rate_limit?
    github_client.rate_limit.remaining > 500
  rescue Octokit::Unauthorized
    false
  end
{'fraction_non_alphanumeric': 0.06862745098039216, 'fraction_numerical': 0.00904977375565611, 'mean_word_length': 3.3795379537953796, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5595903', 'n_tokens_mistral': 455, 'n_tokens_neox': 439, 'n_words': 88}