<commit_before>require "erb"
require "json"
require "pathname"
require "serverkit/errors/non_existent_path_error"
require "tempfile"
require "yaml"

module Serverkit
  module Loaders
    class BaseLoader
      YAML_EXTNAMES = [".yaml", ".yml"]

      # @param [String] path
      def initialize(path)
        @path = path
      end

      def load
        case
        when !pathname.exist?
          raise Errors::NonExistentPathError, pathname
        when has_directory_path?
          load_from_directory
        when has_erb_path?
          load_from_erb
        else
          load_from_data
        end
      end

      private

      # @return [Binding]
      def binding_for_erb
        TOPLEVEL_BINDING
      end

      # @note For override
      def create_empty_loadable
        loaded_class.new({})
      end

      # @return [String]
      def execute
        `#{pathname}`
      end

      # @return [String]
      def expand_erb
        ERB.new(pathname.read).result(binding_for_erb)
      end

      # @return [String]
      def expanded_erb_path_suffix
        "." + pathname.basename(".erb").to_s.split(".", 2).last
      end

      # @note Memoizing to prevent GC
      # @return [Tempfile]
      def expanded_erb_tempfile
        @expanded_erb_tempfile ||= Tempfile.new(["", expanded_erb_path_suffix]).tap do |tempfile|
          tempfile << expand_erb
          tempfile.close
          tempfile
        end
      end

      def has_directory_path?
        pathname.directory?
      end
<commit_msg>Change ERB trim_option from nil to "-"<commit_after>require "erb"
require "json"
require "pathname"
require "serverkit/errors/non_existent_path_error"
require "tempfile"
require "yaml"

module Serverkit
  module Loaders
    class BaseLoader
      YAML_EXTNAMES = [".yaml", ".yml"]

      # @param [String] path
      def initialize(path)
        @path = path
      end

      def load
        case
        when !pathname.exist?
          raise Errors::NonExistentPathError, pathname
        when has_directory_path?
          load_from_directory
        when has_erb_path?
          load_from_erb
        else
          load_from_data
        end
      end

      private

      # @return [Binding]
      def binding_for_erb
        TOPLEVEL_BINDING
      end

      # @note For override
      def create_empty_loadable
        loaded_class.new({})
      end

      # @return [String]
      def execute
        `#{pathname}`
      end

      # @return [String]
      def expand_erb
        ERB.new(pathname.read, nil, "-").result(binding_for_erb)
      end

      # @return [String]
      def expanded_erb_path_suffix
        "." + pathname.basename(".erb").to_s.split(".", 2).last
      end

      # @note Memoizing to prevent GC
      # @return [Tempfile]
      def expanded_erb_tempfile
        @expanded_erb_tempfile ||= Tempfile.new(["", expanded_erb_path_suffix]).tap do |tempfile|
          tempfile << expand_erb
          tempfile.close
          tempfile
        end
      end

      def has_directory_path?
        pathname.directory?
      end

{'fraction_non_alphanumeric': 0.0831979200519987, 'fraction_numerical': 0.0006499837504062399, 'mean_word_length': 1.9681774349083896, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2539331', 'n_tokens_mistral': 977, 'n_tokens_neox': 942, 'n_words': 216}