<commit_before># == Schema Information
#
# Table name: trade_validation_rules
#
#  id                :integer          not null, primary key
#  valid_values_view :string(255)
#  type              :string(255)      not null
#  created_at        :datetime         not null
#  updated_at        :datetime         not null
#  format_re         :string(255)
#  run_order         :integer          not null
#  column_names      :string(255)
#  is_primary        :boolean          default(TRUE), not null
#  scope             :hstore
#

class Trade::PresenceValidationRule < Trade::ValidationRule
  #validates :column_names, :uniqueness => true

  def error_message
    column_names.join(', ') + ' cannot be blank'
  end

  private
  # Returns records where the specified columns are NULL.
  # In case more than one column is specified, predicates are combined
  # using AND.
  def matching_records(table_name)
    s = Arel::Table.new(table_name)
    arel_nodes = column_names.map{|c| s[c].eq(nil)}
    conditions = arel_nodes.shift
    arel_nodes.each{ |n| conditions = conditions.and(n) }
    Trade::SandboxTemplate.select('*').from(table_name).where(conditions)
  end
end<commit_msg>fixed presence validation to check for empty string as well<commit_after># == Schema Information
#
# Table name: trade_validation_rules
#
#  id                :integer          not null, primary key
#  valid_values_view :string(255)
#  type              :string(255)      not null
#  created_at        :datetime         not null
#  updated_at        :datetime         not null
#  format_re         :string(255)
#  run_order         :integer          not null
#  column_names      :string(255)
#  is_primary        :boolean          default(TRUE), not null
#  scope             :hstore
#

class Trade::PresenceValidationRule < Trade::ValidationRule
  #validates :column_names, :uniqueness => true

  def error_message
    column_names.join(', ') + ' cannot be blank'
  end

  private
  # Returns records where the specified columns are NULL.
  # In case more than one column is specified, predicates are combined
  # using AND.
  def matching_records(table_name)
    s = Arel::Table.new(table_name)
    arel_nodes = column_names.map{|c| s[c].eq(nil).or(s[c].eq(''))}
    conditions = arel_nodes.shift
    arel_nodes.each{ |n| conditions = conditions.and(n) }
    Trade::SandboxTemplate.select('*').from(table_name).where(conditions)
  end
end
{'fraction_non_alphanumeric': 0.09752066115702479, 'fraction_numerical': 0.009917355371900827, 'mean_word_length': 2.7476780185758516, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '517993', 'n_tokens_mistral': 746, 'n_tokens_neox': 717, 'n_words': 196}