<commit_before>module RDB
  class Reader
    class << self
      def read_file(rdb_file, options = {})
        File.open(rdb_file, 'rb') do |rdb|
          read(rdb, options)
        end
      end

      def read(rdb, options = {})
        rdb_version = read_rdb_version(rdb)

        state = ReaderState.new(options[:callbacks])
        callbacks = state.callbacks

        callbacks.start_rdb(rdb_version)

        loop do
          state.key_type_id = rdb.readbyte

          case state.key_type_id
          when Opcode::EXPIRETIME_MS
            state.key_expiration = rdb.read(8).unpack('Q').first
            state.key_type_id = rdb.readbyte

          when Opcode::EXPIRETIME
            state.key_expiration = rdb.read(4).unpack('L').first * 1000
            state.key_type_id = rdb.readbyte

          when Opcode::SELECTDB
            callbacks.end_database(state.database) unless state.database.nil?
            state.database, = *read_length(rdb)
            callbacks.start_database(state.database)
            next

          when Opcode::EOF
            callbacks.end_database(state.database) unless state.database.nil?
            callbacks.end_rdb()
            break
          end

          state.key = read_string(rdb)

          if !callbacks.respond_to?(:accept_object?) || callbacks.accept_object?(state)
            read_object(rdb, state)
            notify_expiration(state) if state.key_expires?
          else
            skip_object(rdb, state)<commit_msg>We do not really need the splat operator here...<commit_after>module RDB
  class Reader
    class << self
      def read_file(rdb_file, options = {})
        File.open(rdb_file, 'rb') do |rdb|
          read(rdb, options)
        end
      end

      def read(rdb, options = {})
        rdb_version = read_rdb_version(rdb)

        state = ReaderState.new(options[:callbacks])
        callbacks = state.callbacks

        callbacks.start_rdb(rdb_version)

        loop do
          state.key_type_id = rdb.readbyte

          case state.key_type_id
          when Opcode::EXPIRETIME_MS
            state.key_expiration = rdb.read(8).unpack('Q').first
            state.key_type_id = rdb.readbyte

          when Opcode::EXPIRETIME
            state.key_expiration = rdb.read(4).unpack('L').first * 1000
            state.key_type_id = rdb.readbyte

          when Opcode::SELECTDB
            callbacks.end_database(state.database) unless state.database.nil?
            state.database = read_length(rdb).first
            callbacks.start_database(state.database)
            next

          when Opcode::EOF
            callbacks.end_database(state.database) unless state.database.nil?
            callbacks.end_rdb()
            break
          end

          state.key = read_string(rdb)

          if !callbacks.respond_to?(:accept_object?) || callbacks.accept_object?(state)
            read_object(rdb, state)
            notify_expiration(state) if state.key_expires?
          else
            skip_object(rdb, state)
{'fraction_non_alphanumeric': 0.09232813018930588, 'fraction_numerical': 0.003985386914646297, 'mean_word_length': 2.002991026919242, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7084182', 'n_tokens_mistral': 932, 'n_tokens_neox': 904, 'n_words': 159}