<commit_before>require 'find'
require 'thread'
require 'yaml'

module Downstreams
  class PackerTemplates
    def self.load(cookbook_path, packer_templates_path)
      inst = new
      inst.load(cookbook_path, packer_templates_path)
      inst
    end

    def initialize
      @packer_template_cookbooks = {}
    end

    def each(&block)
      @packer_template_cookbooks.each(&block)
    end

    def load(cookbook_path, packer_templates_path)
      packer_templates(packer_templates_path).each do |filename|
        parsed = YAML.load_file(filename)
        Array(parsed['provisioners']).each do |provisioner|
          next unless provisioner['type'] =~ /chef/
          next if Array(provisioner.fetch('run_list', [])).empty?
          key = PackerTemplate.new(filename)
          @packer_template_cookbooks[key] = find_cookbooks(
            provisioner['run_list'], cookbook_path
          )
        end
      end
    end

    private

    def packer_templates(packer_templates_path)
      packer_templates_path.map do |packer_templates_dir|
        Dir.glob(File.join(packer_templates_dir, '*.yml')).select do |f|
          packer_template?(f)
        end
      end.flatten.compact.sort
    end

    def packer_template?(filename)
      parsed = YAML.load_file(filename)
      %w(variables builders provisioners).all? { |k| parsed.include?(k) }
    rescue => e
      $stderr.puts "ERROR: #{e}\n#{e.backtrace.join("\n")}"
      false
    end

    def find_cookbooks(run_list, cookbook_path)
      Array(run_list).map do |cb|
        find_dependency_cookbooks(
          cb.gsub(/^recipe\[/, '').delete(']'),
          cookbook_path
        )
      end.flatten.sort.uniq
    end

    def find_dependency_cookbooks(cookbook_name, cookbook_path)
      deps = [cookbook_name]
      sem = Mutex.new
      cookbook_path.each do |cookbooks_dir|
        recipes_dir = File.join(cookbooks_dir, cookbook_name, 'recipes')
        next unless File.exist?(recipes_dir)
        Dir.glob(File.join(recipes_dir, '*.rb')) do |recipe_rb|
          sem.synchronize do
            begin
              @included_recipes = []
              instance_eval(File.read(recipe_rb))
              deps += @included_recipes
            rescue => e
              $stderr.puts "ERROR:#{recipe_rb.sub("#{cookbooks_dir}/", '')}: #{e}"
            end<commit_msg>Ensure only cookbook names retained from scanned recipe strings<commit_after>require 'find'
require 'thread'
require 'yaml'

module Downstreams
  class PackerTemplates
    def self.load(cookbook_path, packer_templates_path)
      inst = new
      inst.load(cookbook_path, packer_templates_path)
      inst
    end

    def initialize
      @packer_template_cookbooks = {}
    end

    def each(&block)
      @packer_template_cookbooks.each(&block)
    end

    def load(cookbook_path, packer_templates_path)
      packer_templates(packer_templates_path).each do |filename|
        parsed = YAML.load_file(filename)
        Array(parsed['provisioners']).each do |provisioner|
          next unless provisioner['type'] =~ /chef/
          next if Array(provisioner.fetch('run_list', [])).empty?
          key = PackerTemplate.new(filename)
          @packer_template_cookbooks[key] = find_cookbooks(
            provisioner['run_list'], cookbook_path
          )
        end
      end
    end

    private

    def packer_templates(packer_templates_path)
      packer_templates_path.map do |packer_templates_dir|
        Dir.glob(File.join(packer_templates_dir, '*.yml')).select do |f|
          packer_template?(f)
        end
      end.flatten.compact.sort
    end

    def packer_template?(filename)
      parsed = YAML.load_file(filename)
      %w(variables builders provisioners).all? { |k| parsed.include?(k) }
    rescue => e
      $stderr.puts "ERROR: #{e}\n#{e.backtrace.join("\n")}"
      false
    end

    def find_cookbooks(run_list, cookbook_path)
      Array(run_list).map do |cb|
        find_dependency_cookbooks(
          cb.gsub(/^recipe\[/, '').delete(']').gsub(/::.*/, ''),
          cookbook_path
        )
      end.flatten.sort.uniq
    end

    def find_dependency_cookbooks(cookbook_name, cookbook_path)
      deps = [cookbook_name]
      sem = Mutex.new
      cookbook_path.each do |cookbooks_dir|
        recipes_dir = File.join(cookbooks_dir, cookbook_name, 'recipes')
        next unless File.exist?(recipes_dir)
        Dir.glob(File.join(recipes_dir, '*.rb')) do |recipe_rb|
          sem.synchronize do
            begin
              @included_recipes = []
              instance_eval(File.read(recipe_rb))
              deps += @included_recipes.map { |n| n.gsub(/::.*/, '') }
            rescue => e
              $stderr.puts "ERROR:#{recipe_rb.sub("#{cookbooks_dir}/", '')}: #{e}"
            end
{'fraction_non_alphanumeric': 0.11430369127516779, 'fraction_numerical': 0.0, 'mean_word_length': 2.5483630952380953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5918781', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1535, 'n_words': 271}