<commit_before>#! /usr/bin/ruby
# encoding: utf-8

# for FileUtils.mkdir_p
require 'fileutils'
# for script options
require 'optparse'

# converts style title to style ID
def title_to_styleID(title)
  styleID = title.downcase
  
  # remove content between last set of parentheses
  styleID = styleID.reverse.sub(/\).*?\( /, '').reverse
  
  # remove content between last set of square brackets
  styleID = styleID.reverse.sub(/\].*?\[ /, '').reverse

  # punctuation to eliminate
  styleID.gsub!('?', '')
  styleID.gsub!('’', '')
  styleID.gsub!('(', '')
  styleID.gsub!(')', '')
  styleID.gsub!('\'', '')
  
  # punctuation to replace
  styleID.gsub!(' ', '-')
  styleID.gsub!('–', '-')
  styleID.gsub!(',', '-')
  styleID.gsub!(':', '-')
  styleID.gsub!('.', '-')
  styleID.gsub!('/', '-')
  styleID.gsub!('&', '-and-')
  styleID.gsub!('+', '-')
  styleID.gsub!(/-{2,}/, '-')
  
  # remove hyphens at ends
  styleID.gsub!(/^-/, '')
  styleID.gsub!(/-$/, '')
  
  # remove diacritics
  styleID.gsub!(/á/i, 'a')
  styleID.gsub!(/à/i, 'a')
  styleID.gsub!(/ä/i, 'a')
  styleID.gsub!(/ã/i, 'a')
  styleID.gsub!(/ą/i, 'a')
  styleID.gsub!(/č/i, 'c')
  styleID.gsub!(/ç/i, 'c')
  styleID.gsub!(/é/i, 'e')
  styleID.gsub!(/è/i, 'e')
  styleID.gsub!(/ê/i, 'e')
  styleID.gsub!(/ë/i, 'e')
  styleID.gsub!(/ę/i, 'e')
  styleID.gsub!(/í/i, 'i')
  styleID.gsub!(/ń/i, 'n')
  styleID.gsub!(/ñ/i, 'n')
  styleID.gsub!(/ó/i, 'o')
  styleID.gsub!(/ö/i, 'o')
  styleID.gsub!(/ß/i, 'ss')
  styleID.gsub!(/ü/i, 'u')
  
  return styleID
end

options = {:directory => nil, :replace => false}

parser = OptionParser.new do|opts|
  opts.banner = "Usage: generate_styles.rb [options]"
  opts.on('-d', '--dir directory', 'Limit script to specified data subdirectory (e.g., "asm")') do |directory|
    options[:directory] = directory;
  end

  opts.on('-r', '--replace [LIMITED_TO]', ["additions", "deletions", "modifications"],
          'Replace styles in "styles" repo, optionally limited to: "[a]dditions", "[d]eletions", "[m]odifications"') do |replace_type|
    options[:replace] = true
    options[:replace_type] = replace_type || ''
  end

  opts.on('-h', '--help', 'Show help') do
    puts opts
    exit
  end
end

parser.parse!

# Print current directory
This_script_dir = File.dirname(File.expand_path(__FILE__))
$stderr.puts "Script:\t#{This_script_dir}"

# Determine directories to parse
data_subdir_paths = []
if options[:directory] != nil
    if File.directory? "#{This_script_dir}/#{options[:directory]}"
      data_subdir_paths.push("#{options[:directory]}")
    else
      $stderr.puts "WARNING: subdirectory '#{options[:directory]}' does not exist"
      abort "Failed"
    end
else
  Dir.foreach(This_script_dir) do |data_subdir|
    if File.file? "#{This_script_dir}/#{data_subdir}/_template.csl"
      data_subdir_paths.push(data_subdir)
    end
  end
end

# determine whether styles can be replaced
replace_styles = false
if (options[:replace] == true)
  
  # check presence style dependents folder
  Dependent_dir_path = File.expand_path("../../styles/dependent", This_script_dir)
  if File.exist? Dependent_dir_path
    replace_styles = true
    
    do_additions = false
    do_deletions = false
    do_modifications = false
    
    case options[:replace_type]
    when "additions"
      do_additions = true
    when "deletions"
      do_deletions = true
    when "modifications"
      do_modifications = true
    else
      do_additions = true
      do_deletions = true
      do_modifications = true
    end
    
    deleted_styles = 0
    copied_styles = 0
  else
    $stderr.puts "WARNING: Can't replace styles. Target directory not found at '#{Dependent_dir_path}'"
    abort "Failed"
  end
end

# start with new empty style directory
generated_style_dir_path = "#{This_script_dir}/generated_styles"
`rm -R '#{generated_style_dir_path}'` if File.exist? generated_style_dir_path
$stderr.puts "Output:\t#{generated_style_dir_path}"
$stderr.puts "\n"
$stderr.puts "Generating styles..."
FileUtils.mkdir_p generated_style_dir_path<commit_msg>Change name of generated style folder to "dependent"<commit_after>#! /usr/bin/ruby
# encoding: utf-8

# for FileUtils.mkdir_p
require 'fileutils'
# for script options
require 'optparse'

# converts style title to style ID
def title_to_styleID(title)
  styleID = title.downcase
  
  # remove content between last set of parentheses
  styleID = styleID.reverse.sub(/\).*?\( /, '').reverse
  
  # remove content between last set of square brackets
  styleID = styleID.reverse.sub(/\].*?\[ /, '').reverse

  # punctuation to eliminate
  styleID.gsub!('?', '')
  styleID.gsub!('’', '')
  styleID.gsub!('(', '')
  styleID.gsub!(')', '')
  styleID.gsub!('\'', '')
  
  # punctuation to replace
  styleID.gsub!(' ', '-')
  styleID.gsub!('–', '-')
  styleID.gsub!(',', '-')
  styleID.gsub!(':', '-')
  styleID.gsub!('.', '-')
  styleID.gsub!('/', '-')
  styleID.gsub!('&', '-and-')
  styleID.gsub!('+', '-')
  styleID.gsub!(/-{2,}/, '-')
  
  # remove hyphens at ends
  styleID.gsub!(/^-/, '')
  styleID.gsub!(/-$/, '')
  
  # remove diacritics
  styleID.gsub!(/á/i, 'a')
  styleID.gsub!(/à/i, 'a')
  styleID.gsub!(/ä/i, 'a')
  styleID.gsub!(/ã/i, 'a')
  styleID.gsub!(/ą/i, 'a')
  styleID.gsub!(/č/i, 'c')
  styleID.gsub!(/ç/i, 'c')
  styleID.gsub!(/é/i, 'e')
  styleID.gsub!(/è/i, 'e')
  styleID.gsub!(/ê/i, 'e')
  styleID.gsub!(/ë/i, 'e')
  styleID.gsub!(/ę/i, 'e')
  styleID.gsub!(/í/i, 'i')
  styleID.gsub!(/ń/i, 'n')
  styleID.gsub!(/ñ/i, 'n')
  styleID.gsub!(/ó/i, 'o')
  styleID.gsub!(/ö/i, 'o')
  styleID.gsub!(/ß/i, 'ss')
  styleID.gsub!(/ü/i, 'u')
  
  return styleID
end

options = {:directory => nil, :replace => false}

parser = OptionParser.new do|opts|
  opts.banner = "Usage: generate_styles.rb [options]"
  opts.on('-d', '--dir directory', 'Limit script to specified data subdirectory (e.g., "asm")') do |directory|
    options[:directory] = directory;
  end

  opts.on('-r', '--replace [LIMITED_TO]', ["additions", "deletions", "modifications"],
          'Replace styles in "styles" repo, optionally limited to: "[a]dditions", "[d]eletions", "[m]odifications"') do |replace_type|
    options[:replace] = true
    options[:replace_type] = replace_type || ''
  end

  opts.on('-h', '--help', 'Show help') do
    puts opts
    exit
  end
end

parser.parse!

# Print current directory
This_script_dir = File.dirname(File.expand_path(__FILE__))
$stderr.puts "Script:\t#{This_script_dir}"

# Determine directories to parse
data_subdir_paths = []
if options[:directory] != nil
    if File.directory? "#{This_script_dir}/#{options[:directory]}"
      data_subdir_paths.push("#{options[:directory]}")
    else
      $stderr.puts "WARNING: subdirectory '#{options[:directory]}' does not exist"
      abort "Failed"
    end
else
  Dir.foreach(This_script_dir) do |data_subdir|
    if File.file? "#{This_script_dir}/#{data_subdir}/_template.csl"
      data_subdir_paths.push(data_subdir)
    end
  end
end

# determine whether styles can be replaced
replace_styles = false
if (options[:replace] == true)
  
  # check presence style dependents folder
  Dependent_dir_path = File.expand_path("../../styles/dependent", This_script_dir)
  if File.exist? Dependent_dir_path
    replace_styles = true
    
    do_additions = false
    do_deletions = false
    do_modifications = false
    
    case options[:replace_type]
    when "additions"
      do_additions = true
    when "deletions"
      do_deletions = true
    when "modifications"
      do_modifications = true
    else
      do_additions = true
      do_deletions = true
      do_modifications = true
    end
    
    deleted_styles = 0
    copied_styles = 0
  else
    $stderr.puts "WARNING: Can't replace styles. Target directory not found at '#{Dependent_dir_path}'"
    abort "Failed"
  end
end

# start with new empty style directory
generated_style_dir_path = "#{This_script_dir}/dependent"
`rm -R '#{generated_style_dir_path}'` if File.exist? generated_style_dir_path
$stderr.puts "Output:\t#{generated_style_dir_path}"
$stderr.puts "\n"
$stderr.puts "Generating styles..."
FileUtils.mkdir_p generated_style_dir_path
{'fraction_non_alphanumeric': 0.1785274562915538, 'fraction_numerical': 0.0009849790691947797, 'mean_word_length': 3.5840857787810383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4694878', 'n_tokens_mistral': 3058, 'n_tokens_neox': 3033, 'n_words': 632}