<commit_before>require 'find'
require "safe_yaml/load"
require 'optparse'
require 'json'


options = {
  output_file_path: nil,
  steplib_info_file: "../steplib.yml"
}
opt_parser = OptionParser.new do |opt|
  opt.banner = "Usage: generate_steplib_json.rb [OPTIONS]"
  opt.separator  ""
  opt.separator  "Options"

  opt.on("-o", "--outputfile OUTPUT_FILE_PATH", "Output JSON file path") do |value|
    options[:output_file_path] = value
  end

  opt.on("-h","--help","help") do
    puts opt_parser
    exit
  end
end
opt_parser.parse!

unless options[:output_file_path]
  puts "[!] Output JSON File Path is missing"
  puts opt_parser
  exit
end

# --- UTILS ---

def json_step_item_from_yaml_hash(yaml_hash)
  return {
    name: yaml_hash["name"]
  }
end

# --- MAIN ---

steplib_data = {
  version: nil,
  steps: {}
}

steplib_info = SafeYAML.load_file(options[:steplib_info_file])
steplib_data[:version] = steplib_info["version"]

Find.find("../steps") do |path|
  if FileTest.directory?(path)
    next
  else
    if match = path.match(/steps\/([a-zA-z0-9-]*)\/([0-9]*\.[0-9]*\.[0-9]*)\/step.yml\z/)
      stepid, stepver = match.captures
      step_version_item = json_step_item_from_yaml_hash(SafeYAML.load_file(path))

      steplib_data[:steps][stepid] = {} unless steplib_data[:steps][stepid]

      steplib_data[:steps][stepid][stepver] = step_version_item
    end
  end
end

puts " steplib_data: #{steplib_data.to_json}"

# write to file
File.open(options[:output_file_path], "w") do |f|
  f.write(steplib_data.to_json)
end<commit_msg>generate json : structure change<commit_after>require 'find'
require "safe_yaml/load"
require 'optparse'
require 'json'


options = {
  output_file_path: nil,
  steplib_info_file: "../steplib.yml"
}
opt_parser = OptionParser.new do |opt|
  opt.banner = "Usage: generate_steplib_json.rb [OPTIONS]"
  opt.separator  ""
  opt.separator  "Options"

  opt.on("-o", "--outputfile OUTPUT_FILE_PATH", "Output JSON file path") do |value|
    options[:output_file_path] = value
  end

  opt.on("-h","--help","help") do
    puts opt_parser
    exit
  end
end
opt_parser.parse!

unless options[:output_file_path]
  puts "[!] Output JSON File Path is missing"
  puts opt_parser
  exit
end

# --- UTILS ---

def json_step_item_from_yaml_hash(yaml_hash)
  return {
    name: yaml_hash["name"]
  }
end

def default_step_data_for_stepid(stepid)
  return  {
    id: stepid,
    versions: {}
  }
end

# --- MAIN ---

steplib_data = {
  version: nil,
  steps: {}
}

steplib_info = SafeYAML.load_file(options[:steplib_info_file])
steplib_data[:version] = steplib_info["version"]

Find.find("../steps") do |path|
  if FileTest.directory?(path)
    next
  else
    if match = path.match(/steps\/([a-zA-z0-9-]*)\/([0-9]*\.[0-9]*\.[0-9]*)\/step.yml\z/)
      stepid, stepver = match.captures
      step_version_item = json_step_item_from_yaml_hash(SafeYAML.load_file(path))

      unless steplib_data[:steps][stepid]
        steplib_data[:steps][stepid] = default_step_data_for_stepid(stepid)
      end

      steplib_data[:steps][stepid][:versions][stepver] = step_version_item
    end
  end
end

puts " steplib_data: #{steplib_data.to_json}"

# write to file
File.open(options[:output_file_path], "w") do |f|
  f.write(steplib_data.to_json)
end
{'fraction_non_alphanumeric': 0.15099540581929555, 'fraction_numerical': 0.004900459418070444, 'mean_word_length': 3.903903903903904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6698461', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1264, 'n_words': 227}