<commit_before>require "tempfile"
require "fileutils"

module SuperModel
  module Marshal
    def path
      @path || raise("Provide a path")
    end
    
    def path=(p)
      @path = p
    end
    
    def klasses
      @klasses ||= []
    end
    
    def load
      return unless path
      return unless File.exist?(path)
      records = []
      File.open(path, "rb") do |file|
        begin
          records = ::Marshal.load(file)
        rescue
          # Lots of errors can occur during
          # marshaling - such as EOF etc
          return false
        end
      end
      records.each {|r| r.class.records << r }
      true
    end
  
    def dump
      return unless path
      tmp_file = Tempfile.new("rbdump")
      tmp_file.binmode
      records  = klasses.map {|k| k.records }.flatten
      ::Marshal.dump(records, tmp_file)
      # Atomic serialization - so we never corrupt the db
      FileUtils.mv(tmp_file.path, path)
      true
    end
    
    extend self
    
    module Model
      def self.included(base)
        Marshal.klasses << base
      end
      
      def marshal_dump
        serializable_hash
      end

      def marshal_load(atts)
        # Can't call load, since class
        # isn't setup properly
        @attributes = atts
      end
    end
  end
end<commit_msg>different API which allows marshal overriding.<commit_after>require "tempfile"
require "fileutils"

module SuperModel
  module Marshal
    def path
      @path || raise("Provide a path")
    end
    
    def path=(p)
      @path = p
    end
    
    def klasses
      @klasses ||= []
    end
    
    def load
      return unless path
      return unless File.exist?(path)
      data = []
      File.open(path, "rb") do |file|
        begin
          data = ::Marshal.load(file)
        rescue
          # Lots of errors can occur during
          # marshaling - such as EOF etc
          return false
        end
      end
      data.each do |klass, records| 
        klass.marshal_records(records)
      end
      true
    end
  
    def dump
      return unless path
      tmp_file = Tempfile.new("rbdump")
      tmp_file.binmode
      data = klasses.inject({}) {|hash, klass|
        hash[klass] = klass.marshal_records
        hash
      }
      ::Marshal.dump(data, tmp_file)
      # Atomic serialization - so we never corrupt the db
      FileUtils.mv(tmp_file.path, path)
      true
    end
    
    extend self
    
    module Model
      def self.included(base)
        base.extend ClassMethods
        Marshal.klasses << base
      end
      
      def marshal_dump
        serializable_hash
      end

      def marshal_load(atts)
        # Can't call load, since class
        # isn't setup properly
        @attributes = atts
      end
      
      module ClassMethods
        def marshal_records(records = nil)
          @records = records if records
          @records
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.073203949608444, 'fraction_numerical': 0.0, 'mean_word_length': 1.6255585344057193, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1650047', 'n_tokens_mistral': 872, 'n_tokens_neox': 858, 'n_words': 258}