<commit_before>require "liquidoc"
require 'yaml'
require 'json'
require 'optparse'
require 'liquid'
require 'asciidoctor'
require 'logger'
require 'csv'
require 'crack/xml'

# Default settings
@base_dir_def = Dir.pwd + '/'
@base_dir = @base_dir_def
@configs_dir = @base_dir + '_configs'
@templates_dir = @base_dir + '_templates/'
@data_dir = @base_dir + '_data/'
@output_dir = @base_dir + '_output/'
@config_file_def = @base_dir + '_configs/cfg-sample.yml'
@config_file = @config_file_def
@attributes_file_def = '_data/asciidoctor.yml'
@attributes_file = @attributes_file_def
@pdf_theme_file = 'theme/pdf-theme.yml'
@fonts_dir = 'theme/fonts/'
@output_filename = 'index'
@attributes = {}

@logger = Logger.new(STDOUT)
@logger.level = Logger::INFO
@logger.formatter = proc do |severity, datetime, progname, msg|
  "#{severity}: #{msg}\n"
end

# ===
# General methods
# ===

# Pull in a semi-structured data file, converting contents to a Ruby hash
def get_data data_file
  case File.extname(data_file)
  when ".yml"
    begin
      return YAML.load_file(data_file)
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when ".json"
    begin
      return JSON.parse(File.read(data_file))
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when ".xml"
    begin
      data = Crack::XML.parse(File.read(data_file))
      return data['root']
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when ".csv"
    output = []
    i = 0
    begin
      CSV.foreach(data_file, headers: true, skip_blanks: true) do |row|
        output[i] = row.to_hash
        i = i+1
      end
      return {data: output}
    rescue
      @logger.error "The CSV format is invalid."
    end
  else
    @logger.error "The data file is an invalid type. Allowed: .yml, .json, .xml, and .csv."
  end
end

# Establish source, template, index, etc details for build jobs from a config file
# TODO This needs to be turned into a Class?
def config_build config_file
  @logger.debug "Using config file #{config_file}."
  validate_file_input(config_file, "config")
  begin
    config = YAML.load_file(config_file)
  rescue
    unless File.exists?(config_file)
      @logger.error "Config file not found."
    else
      @logger.error "Problem loading config file. Exiting."
    end
    raise "An exception has occurred."
  end
  validate_config_structure(config)
  if config['compile']
    for src in config['compile']
      data = @base_dir + src['data']
      for cfgn in src['builds']
        template = @base_dir + cfgn['template']
        unless cfgn['output'] == "STDOUT" or @output_type == "STDOUT"
          output = @base_dir + cfgn['output']
        else
          output = "STDOUT"
        end
        liquify(data, template, output)
      end
    end
  end
  if config['publish']
    begin
      for pub in config['publish']
        for bld in pub['builds']
          if bld['publish']
            publish(pub, bld)
          else
            @logger.warn "Publish build for '#{index}' backend '#{backend}' disabled."
          end
        end
      end
    rescue Exception => ex
      @logger.error "Error during publish action. #{ex}"
    end
  end
end

# Verify files exist
def validate_file_input file, type
  @logger.debug "Validating input file for #{type} file #{file}"
  error = false
  unless file.is_a?(String) and !file.nil?
    error = "The #{type} file (#{file}) is not valid."
  else
    unless File.exists?(file)
      error = "The #{type} file (#{file}) was not found."
    end
  end
  unless error
    @logger.debug "Input file validated for #{type} file #{file}."
  else
    @logger.error error
    raise "An exception has occurred."
  end
end

def validate_config_structure config
  unless config.is_a? Hash
    @logger.error "The configuration file is not properly structured."
    raise
  else
    unless config['publish'] or config['compile']
      @logger.error "Config file must have at least one top-level section named 'publish:' or 'compile:'."
      exit
    end
  end
# TODO More validation needed
end

# ===
# Liquify BUILD methods
# ===

# Parse given data using given template, saving to given filename
def liquify data_file, template_file, output_file
  @logger.debug "Executing... liquify parsing operation on data file: #{data_file}, template #{template_file}, to #{output_file}."
  validate_file_input(data_file, "data")
  validate_file_input(template_file, "template")
  data = get_data(data_file) # gathers the data
  begin
    template = File.read(template_file) # reads the template file
    template = Liquid::Template.parse(template) # compiles template
    rendered = template.render(data) # renders the output
  rescue Exception => ex
    @logger.error "Problem rendering Liquid template. #{template_file}\n" \
    "#{ex.class} thrown. #{ex.message}"
    raise "An exception has occurred."
  end
  unless @output_type == "STDOUT"
    begin
      Dir.mkdir(@output_dir) unless File.exists?(@output_dir)
      File.open(output_file, 'w') { |file| file.write(rendered) } # saves file
    rescue Exception => ex
      @logger.error "Failed to save output.\n#{ex.class} #{ex.message}"
    end
    if File.exists?(output_file)
      @logger.info "File built: #{File.basename(output_file)}"
    else
      @logger.error "Hrmp! File not built."
    end
  else # if stdout
    puts "========\nOUTPUT: Rendered with template #{template_file}:\n\n#{rendered}\n"
  end
end

# Copy images and other assets into output dir for HTML operations
def copy_assets src, dest
  if @recursive
    dest = "#{dest}/#{src}"
    recursively = "Recursively c"<commit_msg>Changes based on Matt's suggestions<commit_after>require "liquidoc"
require 'yaml'
require 'json'
require 'optparse'
require 'liquid'
require 'asciidoctor'
require 'logger'
require 'csv'
require 'crack/xml'

# Default settings
@base_dir_def = Dir.pwd + '/'
@base_dir = @base_dir_def
@configs_dir = @base_dir + '_configs'
@templates_dir = @base_dir + '_templates/'
@data_dir = @base_dir + '_data/'
@output_dir = @base_dir + '_output/'
@config_file_def = @base_dir + '_configs/cfg-sample.yml'
@config_file = @config_file_def
@attributes_file_def = '_data/asciidoctor.yml'
@attributes_file = @attributes_file_def
@pdf_theme_file = 'theme/pdf-theme.yml'
@fonts_dir = 'theme/fonts/'
@output_filename = 'index'
@attributes = {}

@logger = Logger.new(STDOUT)
@logger.level = Logger::INFO
@logger.formatter = proc do |severity, datetime, progname, msg|
  "#{severity}: #{msg}\n"
end

# ===
# General methods
# ===

# Pull in a semi-structured data file, converting contents to a Ruby hash
def get_data data_file
  case File.extname(data_file)
  when ".yml"
    begin
      return YAML.load_file(data_file)
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when ".json"
    begin
      return JSON.parse(File.read(data_file))
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when ".xml"
    begin
      data = Crack::XML.parse(File.read(data_file))
      return data['root']
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when ".csv"
    output = []
    i = 0
    begin
      CSV.foreach(data_file, headers: true, skip_blanks: true) do |row|
        output[i] = row.to_hash
        i = i+1
      end
      return {data: output}
    rescue
      @logger.error "The CSV format is invalid."
    end
  else
    @logger.error "The data file is an invalid type. Allowed: .yml, .json, .xml, and .csv."
  end
end

# Establish source, template, index, etc details for build jobs from a config file
# TODO This needs to be turned into a Class?
def config_build config_file
  @logger.debug "Using config file #{config_file}."
  validate_file_input(config_file, "config")
  begin
    config = YAML.load_file(config_file)
  rescue
    unless File.exists?(config_file)
      @logger.error "Config file not found."
    else
      @logger.error "Problem loading config file. Exiting."
    end
    raise "Could not load #{config_file}"
  end
  validate_config_structure(config)
  if config['compile']
    for src in config['compile']
      data = @base_dir + src['data']
      for cfgn in src['builds']
        template = @base_dir + cfgn['template']
        unless cfgn['output'] == "STDOUT" or @output_type == "STDOUT"
          output = @base_dir + cfgn['output']
        else
          output = "STDOUT"
        end
        liquify(data, template, output)
      end
    end
  end
  if config['publish']
    begin
      for pub in config['publish']
        for bld in pub['builds']
          if bld['publish']
            publish(pub, bld)
          else
            @logger.warn "Publish build for '#{index}' backend '#{backend}' disabled."
          end
        end
      end
    rescue Exception => ex
      @logger.error "Error during publish action. #{ex}"
    end
  end
end

# Verify files exist
def validate_file_input file, type
  @logger.debug "Validating input file for #{type} file #{file}"
  error = false
  unless file.is_a?(String) and !file.nil?
    error = "The #{type} file (#{file}) is not valid."
  else
    unless File.exists?(file)
      error = "The #{type} file (#{file}) was not found."
    end
  end
  unless error
    @logger.debug "Input file validated for #{type} file #{file}."
  else
    @logger.error
    raise "Could not validate file input: #{error}"
  end
end

def validate_config_structure config
  unless config.is_a? Hash
    message =  "The configuration file is not properly structured; it is not a Hash"
    @logger.error message
    raise message
  else
    unless config['publish'] or config['compile']
      raise "Config file must have at least one top-level section named 'publish:' or 'compile:'."
    end
  end
# TODO More validation needed
end

# ===
# Liquify BUILD methods
# ===

# Parse given data using given template, saving to given filename
def liquify data_file, template_file, output_file
  @logger.debug "Executing... liquify parsing operation on data file: #{data_file}, template #{template_file}, to #{output_file}."
  validate_file_input(data_file, "data")
  validate_file_input(template_file, "template")
  data = get_data(data_file) # gathers the data
  begin
    template = File.read(template_file) # reads the template file
    template = Liquid::Template.parse(template) # compiles template
    rendered = template.render(data) # renders the output
  rescue Exception => ex
    message = "Problem rendering Liquid template. #{template_file}\n" \
      "#{ex.class} thrown. #{ex.message}"
    @logger.error message
    raise message
  end
  unless @output_type == "STDOUT"
    begin
      Dir.mkdir(@output_dir) unless File.exists?(@output_dir)
      File.open(output_file, 'w') { |file| file.write(rendered) } # saves file
    rescue Exception => ex
      @logger.error "Failed to save output.\n#{ex.class} #{ex.message}"
    end
    if File.exists?(output_file)
      @logger.info "File built: #{File.basename(output_file)}"
    else
      @logger.error "Hrmp! File not built."
    end
  else # if stdout
    puts "========\nOUTPUT: Rendered with template #{template_file}:\n\n#{rendered}\n"
  end
end

# Copy images and other assets into output dir for HTML operations
def copy_assets src, dest
  if @recursive
    dest = "#{dest}/#{src}"
    recursively = "Recursively c"
{'fraction_non_alphanumeric': 0.11777431637244722, 'fraction_numerical': 0.00034614053305642093, 'mean_word_length': 3.208667152221413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5237799', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3610, 'n_words': 1148}