<commit_before>require 'fileutils'

# lib
require 'turntables/transaction'
require 'turntables/db_registry'
require 'turntables/sql_modules/version_histories_sql'
require 'turntables/constants/repository_constants'

module Turntables
# @author <NAME>
# A turntables repository. 
class Repository
  include RepositoryConstants

  def initialize
    @transactions = Array.new
  end

  # @param location is the location of the sql repository for now (a directory
  #   for now).
  def register(location)
    @relative_dir = location
    @sequential_dir = "#{@relative_dir}/#{SeqDir}/*"
    @monolithic_dir = "#{@relative_dir}/#{MonoDir}/*"

    # Other init stuff here
    init_sequential_transactions 
  end

  # Function to call in order to make the database. 
  # 
  # TODO: Here, it should detect in what state the current database is in, and
  # go from there. In other words, whether it can skip sequential database
  # transactions by loading a monolithic one to exclude previous transactions.
  #
  # TODO: we need the version histories table, and
  # logic on what to actually execute over here
  def make!
    version = latest_version
    @transactions.each do |transaction| 
      DbRegistry.instance.execute_batch(transaction.data)
    end
  end

  attr_accessor :sequential_dir
  attr_accessor :monolithic_dir
  attr_accessor :relative_dir
  # Array<Turntables::Transaction>
  attr_accessor :sequential_transactions

private 

  # Get the latest database version from within the database
  # @return a Fixnum denoting the latest version retrieved from the database.
  #   A ':fresh' is returned if the database is entirely new.
  def latest_version
    DbRegistry.instance.table_exists? VersionHistorySql::TableName
  end

  # Create the version history table
  def create_version_history_table

  end

  # Find all the transactions that are to be processed sequentially
  def init_sequential_transactions
    sequential_files.each do |path| 
      data        = File.open(path).read
      filename    = path.split(/\//).last
      @transactions.push Transaction.new(data,filename)
    end
    nil
  end

  # Get the sequential transactional files
  def sequential_files; Dir[@sequential_dir] end
  # Get the monolithic transactional files
  def monolithic_files; Dir[@monolithic_dir] end

end
end
<commit_msg>Small refactoring of the repository class.<commit_after>require 'fileutils'

# lib
require 'turntables/transaction'
require 'turntables/db_registry'
require 'turntables/sql_modules/version_histories_sql'
require 'turntables/constants/repository_constants'

module Turntables
# @author <NAME>
# A turntables repository. 
class Repository
  include RepositoryConstants

  def initialize
    @transactions = Array.new
  end

  # @param location is the location of the sql repository for now (a directory
  #   for now).
  def register(location)
    @relative_dir = location
    @sequential_dir = "#{@relative_dir}/#{SeqDir}/*"
    @monolithic_dir = "#{@relative_dir}/#{MonoDir}/*"

    # Other init stuff here
    init_sequential_transactions 
  end

  # Function to call in order to make the database. 
  # 
  # TODO: Here, it should detect in what state the current database is in, and
  # go from there. In other words, whether it can skip sequential database
  # transactions by loading a monolithic one to exclude previous transactions.
  #
  # TODO: we need the version histories table, and
  # logic on what to actually execute over here
  def make!
    version = latest_version
    @transactions.each do |transaction| 
      DbRegistry.instance.execute_batch(transaction.data)
    end
  end

  attr_accessor :sequential_dir
  attr_accessor :monolithic_dir
  attr_accessor :relative_dir
  # Array<Turntables::Transaction>
  attr_accessor :sequential_transactions

private 

  # Get the latest database version from within the database
  # @return a Fixnum denoting the latest version retrieved from the database.
  #   A ':fresh' is returned if the database is entirely new.
  def latest_version
    DbRegistry.instance.table_exists? VersionHistorySql::TableName
  end

  # Create the version history table
  def create_version_history_table

  end

  # Find all the transactions that are to be processed sequentially
  def init_sequential_transactions
    sequential_files.each do |path| 
      data        = File.open(path).read
      filename    = path.split(/\//).last
      @transactions.push Transaction.new(data,filename)
    end
    nil
  end

  # Get the sequential transactional files
  def sequential_files
    get_files_in_dir(@sequential_dir)
  end
  # Get the monolithic transactional files
  def monolithic_files
    get_files_in_dir(@monolithic_dir)
  end

  # Return the files that are in the given directory
  # @param path is the path to look for files in the directory
  # @return Array<String> of files sorted by stringnum_comparison predicate
  def get_files_in_dir(path)
    Dir[path].sort!{|e1,e2| stringnum_comparison(e1,e2)}
  end

  # Compare two strings with each other 
  def stringnum_comparison(a,b)
    extract_digits(a) <=> extract_digits(b)
  end

  # Extract the numbers from the string, and convert to Fixnum
  # @param string is the string to extract the numbers from
  # @return the number that was found in the string
  def extract_digits(string)
    string.gsub(/\D/,'').to_i
  end

end
end

{'fraction_non_alphanumeric': 0.06666666666666667, 'fraction_numerical': 0.0007469654528478057, 'mean_word_length': 3.873521383075523, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1526682', 'n_tokens_mistral': 1619, 'n_tokens_neox': 1517, 'n_words': 531}