<commit_before># -*- coding: utf-8 -*-
require 'webrick'

require 'thor'

require "rubygems"
require "rubygems/mirror/command"
require "rubygems/mirror/fetcher"

module Rubygems
  module Mirror
    module Command
      class CLI < Thor

        DEFAULT_CONFIG = [{
              "from" => "http://production.s3.rubygems.org",
              "to" => File.join(Gem.user_home, '.gem', "rubygems"),
              "parallelism" => 10,
          }]

        BASE_FILES = [
          "latest_specs.#{Gem.marshal_version}.gz",
          "Marshal.#{Gem.marshal_version}.Z",
          "yaml",
        ]

        GEMSPECS_DIR = "quick/Marshal.#{Gem.marshal_version}/"

        desc "fetch", "fetch all the necessary files to the server."
        def fetch
          fetch_allgems
          fetch_basefile
          fetch_gemspecs
        end

        desc "server", "start mirror server."
        def server(port = 4000)
          WEBrick::HTTPServer.new(:DocumentRoot => to, :Port => port).start
        end

        desc "fetch_allgems", "fetch only gems."
        def fetch_allgems
          config_file
          say "fetch_allgems start!", :GREEN
          Gem::Commands::MirrorCommand.new.execute
          say "fetch_allgems end!", :GREEN
        end

        desc "fetch_gemspecs", "fetch only gemspec files."
        def fetch_gemspecs
          say "fetch gemspecs start!", :GREEN
          Dir::foreach(File.join(to,'gems')) do |filename|
            next if filename == "." or filename == ".."
            gem_name = File.basename filename, ".gem"<commit_msg>Add base file.<commit_after># -*- coding: utf-8 -*-
require 'webrick'

require 'thor'

require "rubygems"
require "rubygems/mirror/command"
require "rubygems/mirror/fetcher"

module Rubygems
  module Mirror
    module Command
      class CLI < Thor

        DEFAULT_CONFIG = [{
              "from" => "http://production.s3.rubygems.org",
              "to" => File.join(Gem.user_home, '.gem', "rubygems"),
              "parallelism" => 10,
          }]

        BASE_FILES = [
          "latest_specs.#{Gem.marshal_version}.gz",
          "Marshal.#{Gem.marshal_version}.Z",
          "yaml",
          "quick/latest_index.rz",
        ]

        GEMSPECS_DIR = "quick/Marshal.#{Gem.marshal_version}/"

        desc "fetch", "fetch all the necessary files to the server."
        def fetch
          fetch_allgems
          fetch_basefile
          fetch_gemspecs
        end

        desc "server [port]", "start mirror server."
        def server(port = 4000)
          WEBrick::HTTPServer.new(:DocumentRoot => to, :Port => port).start
        end

        desc "fetch_allgems", "fetch only gems."
        def fetch_allgems
          config_file
          say "fetch_allgems start!", :GREEN
          Gem::Commands::MirrorCommand.new.execute
          say "fetch_allgems end!", :GREEN
        end

        desc "fetch_gemspecs", "fetch only gemspec files."
        def fetch_gemspecs
          say "fetch gemspecs start!", :GREEN
          Dir::foreach(File.join(to,'gems')) do |filename|
            next if filename == "." or filename == ".."
            gem_name = File.basename filename, ".gem"
{'fraction_non_alphanumeric': 0.12274596646630813, 'fraction_numerical': 0.00506168933881683, 'mean_word_length': 2.0, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7371831', 'n_tokens_mistral': 996, 'n_tokens_neox': 939, 'n_words': 221}