<commit_before>#!/usr/bin/env ruby
require 'liquid'
require 'yaml'

# Open the properties file
properties_file = File.open "properties.yaml"
properties_yaml = properties_file.read
puts properties_yaml
puts
# Ask the user if the properties are correct
puts "Are these properties correct?"
reply = gets
if reply[0] != "y" then
  puts "Aborting.  Edit properties.yaml to make changes."
  exit
end

# Load the properties
properties = YAML::load(properties_yaml)

# process the files
files = [ "pom.xml" ]
files.each do |file|
  puts "Processing #{file}..."
  # read the file
  template = File.open(file, "r").read
  # process the file
  rendering = Liquid::Template.parse(template).render properties
  # write the file
  File.open(file, "w").write(rendering)
end

puts "Successful.  Cleaning up..."
files_to_delete = [ "init.rb", "README" ]
files_to_delete.each do |file|<commit_msg>More flexible templating system that can process filenames<commit_after>#!/usr/bin/env ruby
require 'liquid'
require 'yaml'
require 'fileutils'
require 'find'

# Open the properties file
properties_file = File.open "properties.yaml"
properties_yaml = properties_file.read
puts properties_yaml
puts
# Ask the user if the properties are correct
puts "Are these properties correct?"
reply = gets
if reply[0] != "y" then
  puts "Aborting.  Edit properties.yaml to make changes."
  exit
end

# Load the properties
properties = YAML::load(properties_yaml)

# Create Liquid filters
module NameFilter
  def as_path(input)
    input.tr('.', '/').tr('-', '')
  end
end
Liquid::Template.register_filter(NameFilter)

# process the files
ignore = [ ".git" ]
Find.find(".") do |path|
  if ignore.include? File.basename(path) then
    Find.prune
  elsif FileTest.directory?(path)
    next
  else
    name = CGI::unescape(path)
    name = Liquid::Template.parse(name).render properties
    puts "Processing #{name}..."
    # read the file
    template = File.open(path, "r").read
    # process the file
    rendering = Liquid::Template.parse(template).render properties
    # write the file
    dir = File.dirname(name)
    if !File.exists? dir then
      puts "Creating #{dir}"
      FileUtils.mkdir_p dir
    end
    File.open(name, "w").write(rendering)
    if name != path then
      File.delete path
    end
  end
end

puts "Successful.  Cleaning up..."
files_to_delete = [ "init.rb", "README" ]
files_to_delete.each do |file|
{'fraction_non_alphanumeric': 0.11862990810359231, 'fraction_numerical': 0.000835421888053467, 'mean_word_length': 3.9483471074380163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6446963', 'n_tokens_mistral': 776, 'n_tokens_neox': 762, 'n_words': 236}