<commit_before>require 'json'
module Resity
  class Container
    # number of diff sets, after there will be a new full snapshot stored,
    # max 65000 (16bit)
    MAX_CHANGESETS = 1000
    STORAGE_VERSION = 1

    attr_accessor :name, :last_timestamp, :format
    attr_reader :io, :header, :logger

    def initialize(filename, format, options = {})
      @format = format.new
      raise "invalid format #{@format.class}" if !(format < Resity::Format::Base)
      @format = format.new
      @logger = options[:logger]
      @name = filename
      @bids = {}
      @asks = {}
      @file = nil
      @io = options[:io]
      unless @io
        ff = filename
        unless File.exists?(ff)
          logger.info "fresh db created at #{ff}" if logger
          f = File.open(ff, 'w')
          f.close
        end
        @io = File.open(ff, "r+b")
        unless @io.flock(File::LOCK_EX)
          logger.warn "could not lock db #{ff}" if logger
        end
      end
      @last_timestamp = nil
      @last_checkpoint = nil

      @io.seek(0)
      if @io.size == 0
        # puts "create new db"
        initialize_file
      else
        # puts "load old db"
        initialize_from_file
        scan_last_timestamp unless options[:readonly]
      end
    end

    def seek_timestamp(timestamp)
      @io.seek(1024)
      while (scan_last_timestamp(timestamp) == :next_checkpoint) do
        raise 'huh?'
      end
    end

    def add_snapshot(timestamp, data)
      if @last_checkpoint == nil || @last_checkpoint.num_changesets >= MAX_CHANGESETS
        add_full_snapshot(timestamp, data)
      else
        add_diff(timestamp, data)
      end
    end

    def data
      @format.data
    end

    private

    def initialize_file
      @header = Frames::StorageHeader.new
      @header.version = STORAGE_VERSION
      @header.type = 20
      @header.name = @name
      @header.json_config = {}.to_json
      @header.last_checkpoint = 0
      write_header
    end

    def initialize_from_file
      @io.seek(0)
      @header = Resity::Frames::StorageHeader.new
      @header.read(@io)
      raise StorageError.new("incompatible container format: #{@header.version}") if @header.version > STORAGE_VERSION
    end


    def add_full_snapshot(timestamp, book)
      # TODO: update old checkpoint.
      prev_block = @last_checkpoint ? @header.last_checkpoint : 0
      
      # puts "add full snapshot"
      @io.seek(0, :END)
      @header.last_checkpoint = @io.pos

      cp = Frames::CheckpointHeader.new
      cp.previous_block = prev_block #TODO block pos!
      cp.next_block = 0
      cp.num_changesets = 0
      cp.write(@io)
      @last_checkpoint = cp
      
      # store data?
      @io.seek(0, :END)
      @format.write_snapshot(@io, timestamp)
      post_write_handling(timestamp)
      write_header # nur wg block pointers?
    end<commit_msg>warmup<commit_after>require 'json'
module Resity
  class Container
    # number of diff sets, after there will be a new full snapshot stored,
    # max 65000 (16bit)
    MAX_CHANGESETS = 1000
    STORAGE_VERSION = 1

    attr_accessor :name, :last_timestamp, :format
    attr_reader :io, :header, :logger

    def initialize(filename, format, options = {})
      @format = format.new
      raise "invalid format #{@format.class}" if !(format < Resity::Format::Base)
      @format = format.new
      @logger = options[:logger]
      @name = filename
      @bids = {}
      @asks = {}
      @file = nil
      @io = options[:io]
      unless @io
        ff = filename
        unless File.exists?(ff)
          logger.info "fresh db created at #{ff}" if logger
          f = File.open(ff, 'w')
          f.close
        end
        @io = File.open(ff, "r+b")
        unless @io.flock(File::LOCK_EX)
          logger.warn "could not lock db #{ff}" if logger
        end
      end
      @last_timestamp = nil
      @last_checkpoint = nil

      @io.seek(0)
      if @io.size == 0
        # puts "create new db"
        initialize_file
      else
        # puts "load old db"
        initialize_from_file
        scan_last_timestamp unless options[:readonly]
      end
    end

    def seek_timestamp(timestamp)
      @io.seek(1024)
      while (scan_last_timestamp(timestamp) == :next_checkpoint) do
        raise 'huh?'
      end
    end

    def add_snapshot(timestamp, data)
      if @last_checkpoint == nil || @last_checkpoint.num_changesets >= MAX_CHANGESETS
        add_full_snapshot(timestamp, data)
      else
        add_diff(timestamp, data)
      end
    end

    def data
      @format.data
    end

    private

    def initialize_file
      @header = Frames::StorageHeader.new
      @header.version = STORAGE_VERSION
      @header.type = 20
      @header.name = @name
      @header.json_config = {}.to_json
      @header.last_checkpoint = 0
      write_header
    end

    def initialize_from_file
      @io.seek(0)
      @header = Resity::Frames::StorageHeader.new
      @header.read(@io)
      raise StorageError.new("incompatible container format: #{@header.version}") if @header.version > STORAGE_VERSION
    end


    def add_full_snapshot(timestamp, book)
      # TODO: update old checkpoint.
      prev_block = @last_checkpoint ? @header.last_checkpoint : 0

      # puts "add full snapshot"
      @io.seek(0, :END)
      @header.last_checkpoint = @io.pos

      cp = Frames::CheckpointHeader.new
      cp.previous_block = prev_block #TODO block pos!
      cp.next_block = 0
      cp.num_changesets = 0
      cp.write(@io)
      @last_checkpoint = cp

      # store data?
      @io.seek(0, :END)
      @format.write_snapshot(@io, timestamp)
      post_write_handling(timestamp)
      write_header # nur wg block pointers?
    end
{'fraction_non_alphanumeric': 0.10057823725249693, 'fraction_numerical': 0.009462064131768004, 'mean_word_length': 2.248719408081958, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2451151', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1765, 'n_words': 449}