<commit_before>require 'starscope/langs/ruby'
require "starscope/datum"
require 'zlib'

LANGS = [StarScope::Lang::Ruby]

class StarScope::DB

  DB_FORMAT = 1

  class NoTableError < StandardError; end

  def initialize
    @dirs = []
    @files = {}
    @tables = {}
  end

  def load(file)
    File.open(file, 'r') do |file|
      Zlib::GzipReader.wrap(file) do |file|
        raise "Database format not recognized" if DB_FORMAT != file.gets.to_i
        @dirs   = load_part(file)
        @files  = load_part(file)
        @tables = load_part(file)
      end
    end
  end

  def save(file)
    File.open(file, 'w') do |file|
      Zlib::GzipWriter.wrap(file) do |file|
        file.puts DB_FORMAT
        save_part(file, @dirs)
        save_part(file, @files)
        save_part(file, @tables)
      end
    end
  end

  def add_dirs(dirs)
    @dirs << dirs
    dirs.each do |dir|
      Dir["#{dir}/**/*"].each do |file|
        add_file(file)
      end
    end
  end

  def update
    @files.keys.each {|f| update_file(f)}
    cur_files = @dirs.each {|d| Dir["#{d}/**/*"]}.flatten
    (cur_files - @files.keys).each {|f| add_file(f)}
  end

  def dump_table(table)
    raise NoTableError if not @tables[table]
    puts "== Table: #{table} =="
    @tables[table].each do |val, data|
      puts "#{val}"
      data.each do |datum|
        print "\t"
        puts datum
      end
    end
  end

  def dump_all
    @tables.keys.each {|tbl| dump_table(tbl)}
  end
<commit_msg>Fix bug, got my array ops mixed up again<commit_after>require 'starscope/langs/ruby'
require "starscope/datum"
require 'zlib'

LANGS = [StarScope::Lang::Ruby]

class StarScope::DB

  DB_FORMAT = 1

  class NoTableError < StandardError; end

  def initialize
    @dirs = []
    @files = {}
    @tables = {}
  end

  def load(file)
    File.open(file, 'r') do |file|
      Zlib::GzipReader.wrap(file) do |file|
        raise "Database format not recognized" if DB_FORMAT != file.gets.to_i
        @dirs   = load_part(file)
        @files  = load_part(file)
        @tables = load_part(file)
      end
    end
  end

  def save(file)
    File.open(file, 'w') do |file|
      Zlib::GzipWriter.wrap(file) do |file|
        file.puts DB_FORMAT
        save_part(file, @dirs)
        save_part(file, @files)
        save_part(file, @tables)
      end
    end
  end

  def add_dirs(dirs)
    @dirs += dirs
    dirs.each do |dir|
      Dir["#{dir}/**/*"].each do |file|
        add_file(file)
      end
    end
  end

  def update
    @files.keys.each {|f| update_file(f)}
    cur_files = @dirs.each {|d| Dir["#{d}/**/*"]}.flatten
    (cur_files - @files.keys).each {|f| add_file(f)}
  end

  def dump_table(table)
    raise NoTableError if not @tables[table]
    puts "== Table: #{table} =="
    @tables[table].each do |val, data|
      puts "#{val}"
      data.each do |datum|
        print "\t"
        puts datum
      end
    end
  end

  def dump_all
    @tables.keys.each {|tbl| dump_table(tbl)}
  end

{'fraction_non_alphanumeric': 0.14093508240834174, 'fraction_numerical': 0.0006727211570803902, 'mean_word_length': 2.3911060433295326, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467670', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1103, 'n_words': 264}