<commit_before>module Monotable
class Server

  # config variables
  attr_reader :port,:host,:options
  attr_accessor :verbose

  # server module instances
  attr_reader :local_store,:router,:cluster_manager,:load_balancer,:global_index

  # options
  #   :store_paths=>["path",...]
  #   :port => Fixnum - TCP port to listen on, default 8080
  #   :host => host to listen on - default "localhost"
  def initialize(options={})
    @options = options
    @verbose = options[:verbose]
    if verbose
      puts "Monotable #{Monotable::VERSION}"
      puts "Startup options: #{options.inspect}"
      puts ""
    end

    @local_store = Monotable::LocalStore.new(options)
    @router = Monotable::Router.new(self)
    @cluster_manager = Monotable::ClusterManager.new(self)
    @load_balancer = Monotable::LoadBalancer.new(self)
    @global_index = Monotable::GlobalIndex.new(self)

    @port = options[:port] || 8080
    @host = options[:host] || 'localhost'

    @cluster_manager.local_server = "#{@host}:#{@port}"

    initialize_new_store if options[:initialize_new_store]
  end

  def initialize_new_store
    local_store.verify_store_is_blank_for_init
    puts "Initializing new multi-store..." if @options[:verbose]
    max_chunk_size = local_store.max_chunk_size
    max_index_block_size = local_store.max_index_block_size
    chunk_starts=[
      "",
      INDEX_KEY_PREFIX*3+FIRST_DATA_KEY,  # for 64meg chunks approx 2^16 records max at this index level
      INDEX_KEY_PREFIX*2+FIRST_DATA_KEY,  # for 64meg chunks approx 2^32 records max at this index level
      INDEX_KEY_PREFIX*1+FIRST_DATA_KEY,  # for 64meg chunks approx 2^48 records max at this index level
      FIRST_DATA_KEY                      # for 64meg chunks approx 2^74 bytes max at this index level
    ]<commit_msg>Can now specify the number of index levels for a store on init.<commit_after>module Monotable
class Server

  # config variables
  attr_reader :port,:host,:options
  attr_accessor :verbose

  # server module instances
  attr_reader :local_store,:router,:cluster_manager,:load_balancer,:global_index

  # options
  #   :store_paths=>["path",...]
  #   :port => Fixnum - TCP port to listen on, default 8080
  #   :host => host to listen on - default "localhost"
  def initialize(options={})
    @options = options
    @verbose = options[:verbose]
    if verbose
      puts "Monotable #{Monotable::VERSION}"
      puts "Startup options: #{options.inspect}"
      puts ""
    end

    @local_store = Monotable::LocalStore.new(options)
    @router = Monotable::Router.new(self)
    @cluster_manager = Monotable::ClusterManager.new(self)
    @load_balancer = Monotable::LoadBalancer.new(self)
    @global_index = Monotable::GlobalIndex.new(self)

    @port = options[:port] || 8080
    @host = options[:host] || 'localhost'

    @cluster_manager.local_server = "#{@host}:#{@port}"

    initialize_new_store if options[:initialize_new_store]
  end

  def initialize_new_store
    local_store.verify_store_is_blank_for_init
    max_chunk_size = local_store.max_chunk_size
    max_index_block_size = local_store.max_index_block_size

    num_index_levels = options[:num_index_levels] || 3
    raise ArgumentError.new("num_index_levels must be >=1") unless num_index_levels >= 1

    chunk_starts=[FIRST_DATA_KEY]
    num_index_levels.times do |level|
      chunk_starts << INDEX_KEY_PREFIX * (level+1) + FIRST_DATA_KEY
    end
    chunk_starts<<""
    chunk_starts.reverse!

    if true #options[:verbose]
      puts "Initializing new multi-store..."
      puts "  #{num_index_levels} index level(s)"
      puts "  #{max_chunk_size} bytes per chunk max"
      n = Math.log(max_chunk_size,2)
      m = num_index_levels
      address_bits = Tools.monotable_address_space_size(max_chunk_size,num_index_levels)
      puts "  Estimated max storage:        2^#{address_bits} bytes (#{Tools.commaize 2**(address_bits-40)} terabytes)"
      puts "  Estimated 'safe' max storage: 2^#{address_bits-2} bytes (#{Tools.commaize 2**(address_bits-42)} terabytes)"
      puts "  Initial chunk: #{chunk_starts.inspect}"
    end

{'fraction_non_alphanumeric': 0.10774493036892255, 'fraction_numerical': 0.01294893720987051, 'mean_word_length': 3.9265944645006017, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1857214', 'n_tokens_mistral': 1381, 'n_tokens_neox': 1319, 'n_words': 294}