<commit_before>class SQLFromFile
  QUERY_SEPARATOR = "-- ruby-sql-query:"

  def initialize(filename)
    queries = File.read(filename).split(QUERY_SEPARATOR).map{|q| q.strip}.select{|q| !q.empty?}
    if queries.size == 1
      @queries = {only: queries.first}
    else
      @queries = queries.inject({}) {|all,one| all[one.lines.first.strip.to_sym] = one.lines[1..-1].join; all}
    end
  end
  
  def queries
    @queries.keys
  end
  
  def query(name=nil, vars = {})
    raise "No queries found" if @queries.size == 0
    raise "Specify which query" if @queries.size > 1 and name.nil?
    raise "Unsupported characters in values: ['\";]" if vars.values.select{|v| v.class == String}.any?{|v| v =~ /[\'\"\;]/}
    query = @queries[name || :only]
    return query if vars.empty?
    vars.inject(query) {|a,kv| k,v=*kv; a = a.gsub("__" + k.to_s.upcase + "__", v.to_s)}
  end
end<commit_msg>allowing character that were disallowed before - makes things more dangerous!<commit_after>class SQLFromFile
  QUERY_SEPARATOR = "-- ruby-sql-query:"

  def initialize(filename)
    queries = File.read(filename).split(QUERY_SEPARATOR).map{|q| q.strip}.select{|q| !q.empty?}
    if queries.size == 1
      @queries = {only: queries.first}
    else
      @queries = queries.inject({}) {|all,one| all[one.lines.first.strip.to_sym] = one.lines[1..-1].join; all}
    end
  end
  
  def queries
    @queries.keys
  end
  
  def query(name=nil, vars = {})
    raise "No queries found" if @queries.size == 0
    raise "Specify which query" if @queries.size > 1 and name.nil?
    # raise "Unsupported characters in values: ['\";]" if vars.values.select{|v| v.class == String}.any?{|v| v =~ /[\'\"\;]/}
    query = @queries[name || :only]
    return query if vars.empty?
    vars.inject(query) {|a,kv| k,v=*kv; a = a.gsub("__" + k.to_s.upcase + "__", v.to_s)}
  end
end
{'fraction_non_alphanumeric': 0.17764578833693304, 'fraction_numerical': 0.005399568034557235, 'mean_word_length': 3.2209567198177678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611797', 'n_tokens_mistral': 707, 'n_tokens_neox': 690, 'n_words': 156}