<commit_before>require_relative '../puppet-check'

# executes diagnostics on data files
class DataParser
  # checks yaml (.yaml/.yml)
  def self.yaml(files)
    require 'yaml'

    files.each do |file|
      # check yaml syntax
      begin
        parsed = YAML.load_file(file)
      rescue StandardError => err
        PuppetCheck.settings[:error_files].push("#{file}:\n#{err.to_s.gsub("(#{file}): ", '')}")
      else
        warnings = []

        # perform some rudimentary hiera checks if data exists and is hieradata
        warnings = hiera(parsed, file) unless parsed.nil? || (File.basename(file) == 'hiera.yaml')

        next PuppetCheck.settings[:warning_files].push("#{file}:\n#{warnings.join("\n")}") unless warnings.empty?
        PuppetCheck.settings[:clean_files].push(file.to_s)
      end
    end
  end

  # checks eyaml (.eyaml/.eyml)
  def self.eyaml(files, public, private)
    require 'openssl'

    # keys specified?
    return warn 'Public X509 and/or Private RSA PKCS7 certs were not specified. EYAML checks will not be executed.' if public.nil? || private.nil?

    # keys exist?
    return warn 'Specified Public X509 and/or Private RSA PKCS7 certs do not exist. EYAML checks will not be executed.' unless File.file?(public) && File.file?(private)

    # setup decryption
    rsa = OpenSSL::PKey::RSA.new(File.read(private))
    x509 = OpenSSL::X509::Certificate.new(File.read(public))<commit_msg>if eyaml inputs are faulty, add eyamls to ignored files<commit_after>require_relative '../puppet-check'

# executes diagnostics on data files
class DataParser
  # checks yaml (.yaml/.yml)
  def self.yaml(files)
    require 'yaml'

    files.each do |file|
      # check yaml syntax
      begin
        parsed = YAML.load_file(file)
      rescue StandardError => err
        PuppetCheck.settings[:error_files].push("#{file}:\n#{err.to_s.gsub("(#{file}): ", '')}")
      else
        warnings = []

        # perform some rudimentary hiera checks if data exists and is hieradata
        warnings = hiera(parsed, file) unless parsed.nil? || (File.basename(file) == 'hiera.yaml')

        next PuppetCheck.settings[:warning_files].push("#{file}:\n#{warnings.join("\n")}") unless warnings.empty?
        PuppetCheck.settings[:clean_files].push(file.to_s)
      end
    end
  end

  # checks eyaml (.eyaml/.eyml)
  def self.eyaml(files, public, private)
    require 'openssl'

    # keys specified?
    if public.nil? || private.nil?
      PuppetCheck.settings[:ignored_files].concat(files)
      return warn 'Public X509 and/or Private RSA PKCS7 certs were not specified. EYAML checks will not be executed.'
    end

    # keys exist?
    unless File.file?(public) && File.file?(private)
      PuppetCheck.settings[:ignored_files].concat(files)
      return warn 'Specified Public X509 and/or Private RSA PKCS7 certs do not exist. EYAML checks will not be executed.'
    end

    # setup decryption
    rsa = OpenSSL::PKey::RSA.new(File.read(private))
    x509 = OpenSSL::X509::Certificate.new(File.read(public))
{'fraction_non_alphanumeric': 0.1264026402640264, 'fraction_numerical': 0.009240924092409241, 'mean_word_length': 3.2450980392156863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3079980', 'n_tokens_mistral': 993, 'n_tokens_neox': 951, 'n_words': 250}