<commit_before>require 'rubygems'
require 'fileutils'

class Gem::Mirror
  autoload :Fetcher, 'rubygems/mirror/fetcher'
  autoload :Pool, 'rubygems/mirror/pool'

  VERSION = '1.0.1'

  SPECS_FILE = "specs.#{Gem.marshal_version}"
  SPECS_FILE_Z = "specs.#{Gem.marshal_version}.gz"

  DEFAULT_URI = 'http://production.cf.rubygems.org/'
  DEFAULT_TO = File.join(Gem.user_home, '.gem', 'mirror')

  RUBY = 'ruby'

  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = nil, retries = nil, skiperror = nil)
    @from, @to = from, to
    @fetcher = Fetcher.new :retries=>retries, :skiperror=>skiperror
    @pool = Pool.new(parallelism || 10)
  end

  def from(*args)
    File.join(@from, *args)
  end

  def to(*args)
    File.join(@to, *args)
  end

  def update_specs
    specz = to(SPECS_FILE_Z)
    @fetcher.fetch(from(SPECS_FILE_Z), specz)
    open(to(SPECS_FILE), 'wb') { |f| f << Gem.gunzip(Gem.read_binary(specz)) }
  end

  def gems
    update_specs unless File.exists?(to(SPECS_FILE))

    gems = Marshal.load(Gem.read_binary(to(SPECS_FILE)))
    gems.map! do |name, ver, plat|
      # If the platform is ruby, it is not in the gem name
      "#{name}-#{ver}#{"-#{plat}" unless plat == RUBY}.gem"
    end
    gems
  end

  def existing_gems
    Dir[to('gems', '*.gem')].entries.map { |f| File.basename(f) }
  end<commit_msg>Update mirror.rb<commit_after>require 'rubygems'
require 'fileutils'

class Gem::Mirror
  autoload :Fetcher, 'rubygems/mirror/fetcher'
  autoload :Pool, 'rubygems/mirror/pool'

  VERSION = '1.0.1'

  SPECS_FILE = "specs.#{Gem.marshal_version}"
  SPECS_FILE_Z = "specs.#{Gem.marshal_version}.gz"

  DEFAULT_URI = 'http://production.cf.rubygems.org/'
  DEFAULT_TO = File.join(Gem.user_home, '.gem', 'mirror')

  RUBY = 'ruby'

  def initialize(from = DEFAULT_URI, to = DEFAULT_TO, parallelism = nil, retries = nil, skiperror = nil)
    @from, @to = from, to
    @fetcher = Fetcher.new :retries => retries, :skiperror => skiperror
    @pool = Pool.new(parallelism || 10)
  end

  def from(*args)
    File.join(@from, *args)
  end

  def to(*args)
    File.join(@to, *args)
  end

  def update_specs
    specz = to(SPECS_FILE_Z)
    @fetcher.fetch(from(SPECS_FILE_Z), specz)
    open(to(SPECS_FILE), 'wb') { |f| f << Gem.gunzip(Gem.read_binary(specz)) }
  end

  def gems
    update_specs unless File.exists?(to(SPECS_FILE))

    gems = Marshal.load(Gem.read_binary(to(SPECS_FILE)))
    gems.map! do |name, ver, plat|
      # If the platform is ruby, it is not in the gem name
      "#{name}-#{ver}#{"-#{plat}" unless plat == RUBY}.gem"
    end
    gems
  end

  def existing_gems
    Dir[to('gems', '*.gem')].entries.map { |f| File.basename(f) }
  end
{'fraction_non_alphanumeric': 0.15805132019338045, 'fraction_numerical': 0.003718854592785422, 'mean_word_length': 3.582623509369676, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794310', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1059, 'n_words': 220}