<commit_before>module ZTK::DSL::Core

  # @author <NAME> <zachary AT jovelabs DOT com>
  # @api private
  module Dataset

    def self.included(base)
      base.class_eval do
        base.send(:extend, ZTK::DSL::Core::Dataset::ClassMethods)
      end
    end

    # @author <NAME> <zachary AT jovelabs DOT com>
    module ClassMethods
      def dataset
        klass = self.to_s.underscore.to_sym
        @@dataset ||= {}
        if @@dataset.key?(klass)
          @@dataset[klass]
        else
          @@dataset[klass] ||= []
        end
      end

      def purge
        @@dataset = nil
      end

      def next_id
        klass = self.to_s.underscore.to_sym
        @@id ||= {}
        @@id[klass] ||= 0

        @@id[klass] += 1
        @@id[klass]
      end

    end

  end
end<commit_msg>ensure we only purge our dataset<commit_after>module ZTK::DSL::Core

  # @author <NAME> <zachary AT jovelabs DOT com>
  # @api private
  module Dataset

    def self.included(base)
      base.class_eval do
        base.send(:extend, ZTK::DSL::Core::Dataset::ClassMethods)
      end
    end

    # @author <NAME> <zachary AT jovelabs DOT com>
    module ClassMethods

      def dataset
        klass = self.to_s.underscore.to_sym
        @@dataset ||= {}
        if @@dataset.key?(klass)
          @@dataset[klass]
        else
          @@dataset[klass] ||= []
        end
      end

      def purge
        if !@@dataset.nil?
          klass = self.to_s.underscore.to_sym
          @@dataset[klass] = []
        end
      end

      def next_id
        klass = self.to_s.underscore.to_sym
        @@id ||= {}
        @@id[klass] ||= 0

        @@id[klass] += 1
        @@id[klass]
      end

    end

  end
end
{'fraction_non_alphanumeric': 0.12346401404330018, 'fraction_numerical': 0.0023405500292568754, 'mean_word_length': 1.7895595432300162, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4010216', 'n_tokens_mistral': 610, 'n_tokens_neox': 570, 'n_words': 130}