<commit_before>require "innodb/cursor"

class Innodb::Page
  PAGE_SIZE = 16384

  # InnoDB Page Type constants from include/fil0fil.h.
  PAGE_TYPE = {
    0     => :ALLOCATED,      # Freshly allocated page
    2     => :UNDO_LOG,       # Undo log page
    3     => :INODE,          # Index node
    4     => :IBUF_FREE_LIST, # Insert buffer free list
    5     => :IBUF_BITMAP,    # Insert buffer bitmap
    6     => :SYS,            # System page
    7     => :TRX_SYS,        # Transaction system data
    8     => :FSP_HDR,        # File space header
    9     => :XDES,           # Extent descriptor page
    10    => :BLOB,           # Uncompressed BLOB page
    11    => :ZBLOB,          # First compressed BLOB page
    12    => :ZBLOB2,         # Subsequent compressed BLOB page
    17855 => :INDEX,          # B-tree node
  }

  PAGE_DIRECTION = {
    1 => :left,
    2 => :right,
    3 => :same_rec,
    4 => :same_page,
    5 => :no_direction,
  }

  def initialize(page)
    @page = page
  end

  def cursor(offset)
    Innodb::Cursor.new(self, offset)
  end

  FIL_HEADER_SIZE   = 38
  FIL_HEADER_START  = 0

  PAGE_HEADER_SIZE  = 36
  PAGE_HEADER_START = FIL_HEADER_START + FIL_HEADER_SIZE

  PAGE_TRAILER_SIZE  = 16
  PAGE_TRAILER_START = PAGE_SIZE - PAGE_TRAILER_SIZE

  FSEG_HEADER_SIZE  = 10
  FSEG_HEADER_START = PAGE_HEADER_START + PAGE_HEADER_SIZE
  FSEG_HEADER_COUNT = 2

  MUM_RECORD_SIZE   = 8
  
  RECORD_BITS_SIZE  = 3
  RECORD_NEXT_SIZE  = 2

  def size_record_header
    case page_header[:format]
    when :compact
      RECORD_BITS_SIZE + RECORD_NEXT_SIZE
    when :redundant
      RECORD_BITS_SIZE + RECORD_NEXT_SIZE + 1
    end
  end

  def size_record_undefined
    case page_header[:format]
    when :compact
      0
    when :redundant
      1
    end
  end

  def pos_records
    FIL_HEADER_SIZE + 
      PAGE_HEADER_SIZE + 
      (FSEG_HEADER_COUNT * FSEG_HEADER_SIZE)
  end

  def pos_infimum
    pos_records + size_record_header + size_record_undefined
  end

  def pos_supremum
    pos_infimum + size_record_header + size_record_undefined + MUM_RECORD_SIZE
  end

  def pos_user_records
    pos_supremum + size_record_header + size_record_undefined + MUM_RECORD_SIZE
  end

  def maybe_undefined(value)
    value == 4294967295 ? nil : value
  end

  def data(offset, length)
    @page[offset...(offset + length)]
  end

  def fil_header
    c = cursor(FIL_HEADER_START)
    @fil_header ||= {
      :checksum   => c.get_uint32,
      :offset     => c.get_uint32,
      :prev       => maybe_undefined(c.get_uint32),
      :next       => maybe_undefined(c.get_uint32),
      :lsn        => c.get_uint64,
      :type       => PAGE_TYPE[c.get_uint16],
      :flush_lsn  => c.get_uint64,
      :space_id   => c.get_uint32,
    }
  end
  alias :fh :fil_header

  def type
    fil_header[:type]
  end

  def prev
    fil_header[:prev]
  end
<commit_msg>Rename @page to @buffer in page.rb<commit_after>require "innodb/cursor"

class Innodb::Page
  PAGE_SIZE = 16384

  # InnoDB Page Type constants from include/fil0fil.h.
  PAGE_TYPE = {
    0     => :ALLOCATED,      # Freshly allocated page
    2     => :UNDO_LOG,       # Undo log page
    3     => :INODE,          # Index node
    4     => :IBUF_FREE_LIST, # Insert buffer free list
    5     => :IBUF_BITMAP,    # Insert buffer bitmap
    6     => :SYS,            # System page
    7     => :TRX_SYS,        # Transaction system data
    8     => :FSP_HDR,        # File space header
    9     => :XDES,           # Extent descriptor page
    10    => :BLOB,           # Uncompressed BLOB page
    11    => :ZBLOB,          # First compressed BLOB page
    12    => :ZBLOB2,         # Subsequent compressed BLOB page
    17855 => :INDEX,          # B-tree node
  }

  PAGE_DIRECTION = {
    1 => :left,
    2 => :right,
    3 => :same_rec,
    4 => :same_page,
    5 => :no_direction,
  }

  def initialize(buffer)
    @buffer = buffer
  end

  def cursor(offset)
    Innodb::Cursor.new(self, offset)
  end

  FIL_HEADER_SIZE   = 38
  FIL_HEADER_START  = 0

  PAGE_HEADER_SIZE  = 36
  PAGE_HEADER_START = FIL_HEADER_START + FIL_HEADER_SIZE

  PAGE_TRAILER_SIZE  = 16
  PAGE_TRAILER_START = PAGE_SIZE - PAGE_TRAILER_SIZE

  FSEG_HEADER_SIZE  = 10
  FSEG_HEADER_START = PAGE_HEADER_START + PAGE_HEADER_SIZE
  FSEG_HEADER_COUNT = 2

  MUM_RECORD_SIZE   = 8
  
  RECORD_BITS_SIZE  = 3
  RECORD_NEXT_SIZE  = 2

  def size_record_header
    case page_header[:format]
    when :compact
      RECORD_BITS_SIZE + RECORD_NEXT_SIZE
    when :redundant
      RECORD_BITS_SIZE + RECORD_NEXT_SIZE + 1
    end
  end

  def size_record_undefined
    case page_header[:format]
    when :compact
      0
    when :redundant
      1
    end
  end

  def pos_records
    FIL_HEADER_SIZE + 
      PAGE_HEADER_SIZE + 
      (FSEG_HEADER_COUNT * FSEG_HEADER_SIZE)
  end

  def pos_infimum
    pos_records + size_record_header + size_record_undefined
  end

  def pos_supremum
    pos_infimum + size_record_header + size_record_undefined + MUM_RECORD_SIZE
  end

  def pos_user_records
    pos_supremum + size_record_header + size_record_undefined + MUM_RECORD_SIZE
  end

  def maybe_undefined(value)
    value == 4294967295 ? nil : value
  end

  def data(offset, length)
    @buffer[offset...(offset + length)]
  end

  def fil_header
    c = cursor(FIL_HEADER_START)
    @fil_header ||= {
      :checksum   => c.get_uint32,
      :offset     => c.get_uint32,
      :prev       => maybe_undefined(c.get_uint32),
      :next       => maybe_undefined(c.get_uint32),
      :lsn        => c.get_uint64,
      :type       => PAGE_TYPE[c.get_uint16],
      :flush_lsn  => c.get_uint64,
      :space_id   => c.get_uint32,
    }
  end
  alias :fh :fil_header

  def type
    fil_header[:type]
  end

  def prev
    fil_header[:prev]
  end

{'fraction_non_alphanumeric': 0.08297395204416078, 'fraction_numerical': 0.0255304467828187, 'mean_word_length': 2.3264486517498564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3261028', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2054, 'n_words': 457}