<commit_before>#!/usr/bin/env ruby

# this script will create SQL insert statements from one or more .properties file arguments
# for use with the io.lumify.core.config.DatabaseConfigurationLoader

table              = ENV['TABLE']              || 'configuration'
environment_column = ENV['ENVIRONMENT_COLUMN'] || 'environment'
version_column     = ENV['VERSION_COLUMN']     || 'version'
key_column         = ENV['KEY_COLUMN']         || 'k'
value_column       = ENV['VALUE_COLUMN']       || 'v'
environment        = ENV['ENVIRONMENT']        || 'dev'
version            = ENV['VERSION']            || '1.0'
key_prefix         = ENV['KEY_PREFIX']         || 'lumify'

puts "-- delete from #{table} where #{environment_column} = '#{environment}'" +
                              " and #{version_column} = '#{version}'" +
                              " and #{key_column} like '#{key_prefix}.%';"

ARGV.each do |arg|
  if File.exist?(arg)
    File.open(arg, 'r') do |file|
      file.each_line do |line|
        next if line.match(/\s*#|^\s*$/)
        line.chomp!
        i = line.index(/:|=/)
        k = "#{key_prefix}.#{line[0..i-1]}"
        v = line[i+1..line.length].gsub("'"){"\\'"}
        puts "insert into #{table}(#{environment_column}, #{version_column}, #{key_column}, #{value_column})" +
                          " values('#{environment}', '#{version}', '#{k}', '#{v}');"
      end
    end
  end
end<commit_msg>only skip comments<commit_after>#!/usr/bin/env ruby

# this script will create SQL insert statements from one or more .properties file arguments
# for use with the io.lumify.core.config.DatabaseConfigurationLoader

table              = ENV['TABLE']              || 'configuration'
environment_column = ENV['ENVIRONMENT_COLUMN'] || 'environment'
version_column     = ENV['VERSION_COLUMN']     || 'version'
key_column         = ENV['KEY_COLUMN']         || 'k'
value_column       = ENV['VALUE_COLUMN']       || 'v'
environment        = ENV['ENVIRONMENT']        || 'dev'
version            = ENV['VERSION']            || '1.0'
key_prefix         = ENV['KEY_PREFIX']         || 'lumify'

puts "-- delete from #{table} where #{environment_column} = '#{environment}'" +
                              " and #{version_column} = '#{version}'" +
                              " and #{key_column} like '#{key_prefix}.%';"

ARGV.each do |arg|
  if File.exist?(arg)
    File.open(arg, 'r') do |file|
      file.each_line do |line|
        next if line.match(/^\s*#|^\s*$/)
        line.chomp!
        i = line.index(/:|=/)
        k = "#{key_prefix}.#{line[0..i-1]}"
        v = line[i+1..line.length].gsub("'"){"\\'"}
        puts "insert into #{table}(#{environment_column}, #{version_column}, #{key_column}, #{value_column})" +
                          " values('#{environment}', '#{version}', '#{k}', '#{v}');"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.18102240896358543, 'fraction_numerical': 0.0035014005602240898, 'mean_word_length': 2.265142857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '887992', 'n_tokens_mistral': 903, 'n_tokens_neox': 918, 'n_words': 193}