<commit_before>require "columns/version"
require "columns/table"
require "columns/regex"
require "columns/raw_data"
require "columns/model_data"
require "columns/model_cleaner"
require "columns/model_writer"
require "columns/extended_content"

# Public: Columns annotate your activerecord models from a `schema.rb` file.
# That could be handy when your application don't use Rails.
module Columns

  # Public: Annotate models.
  #
  # schema_dir - String directory (absolute or relative) to find the
  #              `schema.rb` file. Default is `./db/`.
  # models_dir - String directory (absolute or relative) to find the
  #              models. Default is `./app/models/`.
  #
  # Returns nothing.
  #
  # Raises SystemExit if the `schema.rb` doesn't exist.
  def self.execute(schema_dir: './db/', models_dir: './app/models/')
    application = Application.new(schema_dir, models_dir)
    application.execute
  end

  private

  # The tool chain.
  class Application

    # Creates a new Application.
    #
    # See Columns.execute for arguments.
    #
    # Raises SystemExit if it can't find the `schema.rb`.
    def initialize(schema_dir, models_dir)
      @models_dir = models_dir
      schema_path = File.expand_path(File.join(schema_dir,'schema.rb'))

      unless File.exists?(schema_path)
        puts "COLUMNS ERROR : #{schema_path} doesn't exist!"
        exit 1
      end
      table = Table.new(File.read(schema_path))

      raw_data_objects = []
      table.names.each do |name|
        raw_data_objects << RawData.new(name, table.content_for(name))
      end

      @model_data_objects = raw_data_objects.map do |object|
        ModelData.new(object)
      end
    end

    # Cleans then writes models.
    #
    # Returns nothing.
    def execute
      writer = ModelWriter.new(path: @models_dir)
      @model_data_objects.each do |object|
        path = File.expand_path(File.join(@models_dir, object.name) + '.rb')
        ModelCleaner.clean(path)
        writer.add_info(object)
      end
    end

  end
end<commit_msg>Add a TODO note<commit_after>require "columns/version"
require "columns/table"
require "columns/regex"
require "columns/raw_data"
require "columns/model_data"
require "columns/model_cleaner"
require "columns/model_writer"
require "columns/extended_content"

# Public: Columns annotate your activerecord models from a `schema.rb` file.
# That could be handy when your application don't use Rails.
module Columns

  # Public: Annotate models.
  #
  # schema_dir - String directory (absolute or relative) to find the
  #              `schema.rb` file. Default is `./db/`.
  # models_dir - String directory (absolute or relative) to find the
  #              models. Default is `./app/models/`.
  #
  # Returns nothing.
  #
  # Raises SystemExit if the `schema.rb` doesn't exist.
  def self.execute(schema_dir: './db/', models_dir: './app/models/')
    application = Application.new(schema_dir, models_dir)
    application.execute
  end

  private

  # The tool chain.
  class Application

    # Creates a new Application.
    #
    # See Columns.execute for arguments.
    #
    # Raises SystemExit if it can't find the `schema.rb`.
    def initialize(schema_dir, models_dir)
      @models_dir = models_dir
      schema_path = File.expand_path(File.join(schema_dir,'schema.rb'))

      unless File.exists?(schema_path)
        puts "COLUMNS ERROR : #{schema_path} doesn't exist!"
        exit 1
      end
      # TODO Table.new should be able to take either a path or an IO.
      #   This removes the need to check for error here (above).
      #   The check will be in Table.
      table = Table.new(File.read(schema_path))

      raw_data_objects = []
      table.names.each do |name|
        raw_data_objects << RawData.new(name, table.content_for(name))
      end

      @model_data_objects = raw_data_objects.map do |object|
        ModelData.new(object)
      end
    end

    # Cleans then writes models.
    #
    # Returns nothing.
    def execute
      writer = ModelWriter.new(path: @models_dir)
      @model_data_objects.each do |object|
        path = File.expand_path(File.join(@models_dir, object.name) + '.rb')
        ModelCleaner.clean(path)
        writer.add_info(object)
      end
    end

  end
end
{'fraction_non_alphanumeric': 0.10068059141046702, 'fraction_numerical': 0.0004693733865289838, 'mean_word_length': 2.8189964157706093, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2303686', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1291, 'n_words': 367}