<commit_before>require 'singleton'
require 'set'

# Public: A singleton class storing all the information about the manifest
# being analysed.
class PuppetLint::Data
  include Singleton

  class << self
    # Internal: Get/Set the full expanded path to the manifest file being
    # checked.
    attr_reader :path, :fullpath, :filename

    # Internal: Get/Set the raw manifest data, split by \n.
    attr_accessor :manifest_lines

    # Internal: Store the tokenised manifest.
    #
    # tokens - The Array of PuppetLint::Lexer::Token objects to store.
    #
    # Returns nothing.
    def tokens=(tokens)
      @tokens = tokens
      @title_tokens = nil
      @resource_indexes = nil
      @class_indexes = nil
      @defined_type_indexes = nil
      @node_indexes = nil
      @function_indexes = nil
      @array_indexes = nil
      @hash_indexes = nil
      @defaults_indexes = nil
    end

    # Public: Get the tokenised manifest.
    #
    # Returns an Array of PuppetLint::Lexer::Token objects.
    def tokens
      calling_method = begin
        caller[0][/`.*'/][1..-2]
      rescue NoMethodError
        caller[1][/`.*'/][1..-2]
      end

      if calling_method == 'check'
        @tokens.dup
      else
        @tokens
      end
    end

    # Internal: Store the path to the manifest file and populate fullpath and
    # filename.
    #
    # val - The path to the file as a String.
    #
    # Returns nothing.
    def path=(val)
      @path = val
      if val.nil?
        @fullpath = nil
        @filename = nil
      else
        @fullpath = File.expand_path(val, ENV['PWD'])
        @filename = File.basename(val)
      end
    end

    # Internal: Retrieve a list of tokens that represent resource titles
    #
    # Returns an Array of PuppetLint::Lexer::Token objects.
    def title_tokens
      @title_tokens ||= Proc.new do
        result = []
        tokens.each_index do |token_idx|
          if tokens[token_idx].type == :COLON
            # gather a list of tokens that are resource titles
            if tokens[token_idx-1].type == :RBRACK
              array_start_idx = tokens.rindex { |r|
                r.type == :LBRACK
              }
              title_array_tokens = tokens[(array_start_idx + 1)..(token_idx - 2)]
              result += title_array_tokens.select { |token|
                {:STRING => true, :NAME => true}.include? token.type
              }
            else
              next_token = tokens[token_idx].next_code_token
              if next_token.type != :LBRACE
                result << tokens[token_idx - 1]
              end
            end
          end
        end
        result
      end.call
    end

    # Internal: Calculate the positions of all resource declarations within the
    # tokenised manifest. These positions only point to the content of the
    # resource declarations, they do not include resource types or titles.
    #
    # Returns an Array of Hashes, each containing:
    #   :start - An Integer position in the `tokens` Array pointing to the
    #            first Token of a resource declaration.
    #   :end   - An Integer position in the `tokens` Array pointing to the last
    #            Token of a resource declaration.
    def resource_indexes
      @resource_indexes ||= Proc.new do
        result = []
        tokens.each_index do |token_idx|
          if tokens[token_idx].type == :COLON
            next_token = tokens[token_idx].next_code_token
            depth = 1
            if next_token && next_token.type != :LBRACE
              tokens[(token_idx + 1)..-1].each_index do |idx|
                real_idx = token_idx + idx + 1
                if tokens[real_idx].type == :LBRACE
                  depth += 1
                elsif {:SEMIC => true, :RBRACE => true}.include? tokens[real_idx].type
                  unless tokens[real_idx].type == :SEMIC && depth > 1
                    depth -= 1
                    if depth == 0
                      result << {
                        :start        => token_idx + 1,
                        :end          => real_idx,
                        :tokens       => tokens[(token_idx + 1)..real_idx],
                        :type         => find_resource_type_token(token_idx),
                        :param_tokens => find_resource_param_tokens(tokens[(token_idx + 1)..real_idx]),
                      }
                      break
                    end
                  end
                end
              end
            end
          end
        end
        result
      end.call
    end

    # Internal: Find the Token representing the type of a resource definition.
    #
    # index - The Integer pointing to the start of the resource in the `tokens`
    #         array.
    #
    # Returns a Token object.
    def find_resource_type_token(index)
      tokens[tokens[0..index].rindex { |token|
               token.type == :LBRACE && token.prev_code_token.type != :QMARK
             }].prev_code_token
    end

    # Internal: Find all the Token objects representing the parameter names in
    # a resource definition.
    #
    # resource_tokens - An Array of Token objects that comprise the resource
    #                   definition.
    #
    # Returns an Array of Token objects.
    def find_resource_param_tokens(resource_tokens)
      resource_tokens.select { |token|
        token.type == :NAME && token.next_code_token.type == :FARROW
      }
    end

    # Internal: Calculate the positions of all class definitions within the
    # `tokens` Array.
    #
    # Returns an Array of Hashes, each containing:
    #   :start  - An Integer position in the `tokens` Array pointing to the
    #             first Token of a class definition.
    #   :end    - An Integer position in the `tokens` Array pointing to the last<commit_msg>Split this to make it easier to read<commit_after>require 'singleton'
require 'set'

# Public: A singleton class storing all the information about the manifest
# being analysed.
class PuppetLint::Data
  include Singleton

  class << self
    # Internal: Get/Set the full expanded path to the manifest file being
    # checked.
    attr_reader :path, :fullpath, :filename

    # Internal: Get/Set the raw manifest data, split by \n.
    attr_accessor :manifest_lines

    # Internal: Store the tokenised manifest.
    #
    # tokens - The Array of PuppetLint::Lexer::Token objects to store.
    #
    # Returns nothing.
    def tokens=(tokens)
      @tokens = tokens
      @title_tokens = nil
      @resource_indexes = nil
      @class_indexes = nil
      @defined_type_indexes = nil
      @node_indexes = nil
      @function_indexes = nil
      @array_indexes = nil
      @hash_indexes = nil
      @defaults_indexes = nil
    end

    # Public: Get the tokenised manifest.
    #
    # Returns an Array of PuppetLint::Lexer::Token objects.
    def tokens
      calling_method = begin
        caller[0][/`.*'/][1..-2]
      rescue NoMethodError
        caller[1][/`.*'/][1..-2]
      end

      if calling_method == 'check'
        @tokens.dup
      else
        @tokens
      end
    end

    # Internal: Store the path to the manifest file and populate fullpath and
    # filename.
    #
    # val - The path to the file as a String.
    #
    # Returns nothing.
    def path=(val)
      @path = val
      if val.nil?
        @fullpath = nil
        @filename = nil
      else
        @fullpath = File.expand_path(val, ENV['PWD'])
        @filename = File.basename(val)
      end
    end

    # Internal: Retrieve a list of tokens that represent resource titles
    #
    # Returns an Array of PuppetLint::Lexer::Token objects.
    def title_tokens
      @title_tokens ||= Proc.new do
        result = []
        tokens.each_index do |token_idx|
          if tokens[token_idx].type == :COLON
            # gather a list of tokens that are resource titles
            if tokens[token_idx-1].type == :RBRACK
              array_start_idx = tokens.rindex { |r|
                r.type == :LBRACK
              }
              title_array_tokens = tokens[(array_start_idx + 1)..(token_idx - 2)]
              result += title_array_tokens.select { |token|
                {:STRING => true, :NAME => true}.include? token.type
              }
            else
              next_token = tokens[token_idx].next_code_token
              if next_token.type != :LBRACE
                result << tokens[token_idx - 1]
              end
            end
          end
        end
        result
      end.call
    end

    # Internal: Calculate the positions of all resource declarations within the
    # tokenised manifest. These positions only point to the content of the
    # resource declarations, they do not include resource types or titles.
    #
    # Returns an Array of Hashes, each containing:
    #   :start - An Integer position in the `tokens` Array pointing to the
    #            first Token of a resource declaration.
    #   :end   - An Integer position in the `tokens` Array pointing to the last
    #            Token of a resource declaration.
    def resource_indexes
      @resource_indexes ||= Proc.new do
        result = []
        tokens.each_index do |token_idx|
          if tokens[token_idx].type == :COLON
            next_token = tokens[token_idx].next_code_token
            depth = 1
            if next_token && next_token.type != :LBRACE
              tokens[(token_idx + 1)..-1].each_index do |idx|
                real_idx = token_idx + idx + 1
                if tokens[real_idx].type == :LBRACE
                  depth += 1
                elsif {:SEMIC => true, :RBRACE => true}.include? tokens[real_idx].type
                  unless tokens[real_idx].type == :SEMIC && depth > 1
                    depth -= 1
                    if depth == 0
                      result << {
                        :start        => token_idx + 1,
                        :end          => real_idx,
                        :tokens       => tokens[(token_idx + 1)..real_idx],
                        :type         => find_resource_type_token(token_idx),
                        :param_tokens => find_resource_param_tokens(tokens[(token_idx + 1)..real_idx]),
                      }
                      break
                    end
                  end
                end
              end
            end
          end
        end
        result
      end.call
    end

    # Internal: Find the Token representing the type of a resource definition.
    #
    # index - The Integer pointing to the start of the resource in the `tokens`
    #         array.
    #
    # Returns a Token object.
    def find_resource_type_token(index)
      lbrace_idx = tokens[0..index].rindex { |token|
        token.type == :LBRACE && token.prev_code_token.type != :QMARK
      }
      tokens[lbrace_idx].prev_code_token
    end

    # Internal: Find all the Token objects representing the parameter names in
    # a resource definition.
    #
    # resource_tokens - An Array of Token objects that comprise the resource
    #                   definition.
    #
    # Returns an Array of Token objects.
    def find_resource_param_tokens(resource_tokens)
      resource_tokens.select { |token|
        token.type == :NAME && token.next_code_token.type == :FARROW
      }
    end

    # Internal: Calculate the positions of all class definitions within the
    # `tokens` Array.
    #
    # Returns an Array of Hashes, each containing:
    #   :start  - An Integer position in the `tokens` Array pointing to the
    #             first Token of a class definition.
    #   :end    - An Integer position in the `tokens` Array pointing to the last
{'fraction_non_alphanumeric': 0.0821106821106821, 'fraction_numerical': 0.0037752037752037753, 'mean_word_length': 1.7278258834542477, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3064772', 'n_tokens_mistral': 3255, 'n_tokens_neox': 3124, 'n_words': 1015}