<commit_before>module DBF
  class Record
    attr_reader :attributes
    delegate :columns, :to => :@table
    
    def initialize(table)
      @table, @data, @memo = table, table.data, table.memo
      initialize_values
      define_accessors
    end
    
    def ==(other)
      other.respond_to?(:attributes) && other.attributes == attributes
    end
    
    def to_a
      columns.map { |column| @attributes[column.name.underscore] }
    end
    
    private
    
    def define_accessors
      columns.each do |column|
        underscored_column_name = column.name.underscore
        unless respond_to?(underscored_column_name)
          self.class.send :define_method, underscored_column_name do
            @attributes[column.name.underscore]
          end
        end
      end
    end
    
    def initialize_values
      @attributes = columns.inject({}) do |hash, column|
        if column.type == 'M'
          starting_block = unpack_data(column.length).to_i
          hash[column.name.underscore] = read_memo(starting_block)
        else
          value = unpack_data(column.length)
          hash[column.name.underscore] = column.type_cast(value)
        end
        hash
      end
    end
  
    def unpack_data(length)
      @data.read(length).unpack("a#{length}").first
    end
  
    def read_memo(start_block)
      return nil if !@table.has_memo_file? || start_block < 1

      @table.memo_file_format == :fpt ? build_fpt_memo(start_block) : build_dbt_memo(start_block)
    end
    
    def build_fpt_memo(start_block)
      @memo.seek(start_block * memo_block_size)
      
      memo_type, memo_size, memo_string = @memo.read(memo_block_size).unpack("NNa56")
      return nil unless memo_type == 1 and memo_size > 0
      
      if memo_size > memo_block_content_size
        memo_string << @memo.read(memo_content_size(memo_size))
      else
        memo_string = memo_string[0, memo_size]
      end
      memo_string
    end
    
    def build_dbt_memo(start_block)
      @memo.seek(start_block * memo_block_size)
      
      case @table.version
      when "83" # dbase iii
        memo_string = ""
        loop do
          block = @memo.read(memo_block_size)
          memo_string << block
          break if block.rstrip.size < memo_block_size<commit_msg>Since FPT "memo_block_size" is variable we should get all remaining chars<commit_after>module DBF
  class Record
    attr_reader :attributes
    delegate :columns, :to => :@table
    
    def initialize(table)
      @table, @data, @memo = table, table.data, table.memo
      initialize_values
      define_accessors
    end
    
    def ==(other)
      other.respond_to?(:attributes) && other.attributes == attributes
    end
    
    def to_a
      columns.map { |column| @attributes[column.name.underscore] }
    end
    
    private
    
    def define_accessors
      columns.each do |column|
        underscored_column_name = column.name.underscore
        unless respond_to?(underscored_column_name)
          self.class.send :define_method, underscored_column_name do
            @attributes[column.name.underscore]
          end
        end
      end
    end
    
    def initialize_values
      @attributes = columns.inject({}) do |hash, column|
        if column.type == 'M'
          starting_block = unpack_data(column.length).to_i
          hash[column.name.underscore] = read_memo(starting_block)
        else
          value = unpack_data(column.length)
          hash[column.name.underscore] = column.type_cast(value)
        end
        hash
      end
    end
  
    def unpack_data(length)
      @data.read(length).unpack("a#{length}").first
    end
  
    def read_memo(start_block)
      return nil if !@table.has_memo_file? || start_block < 1

      @table.memo_file_format == :fpt ? build_fpt_memo(start_block) : build_dbt_memo(start_block)
    end
    
    def build_fpt_memo(start_block)
      @memo.seek(start_block * memo_block_size)
      
      memo_type, memo_size, memo_string = @memo.read(memo_block_size).unpack("NNa*")
      return nil unless memo_type == 1 and memo_size > 0
      
      if memo_size > memo_block_content_size
        memo_string << @memo.read(memo_content_size(memo_size))
      else
        memo_string = memo_string[0, memo_size]
      end
      memo_string
    end
    
    def build_dbt_memo(start_block)
      @memo.seek(start_block * memo_block_size)
      
      case @table.version
      when "83" # dbase iii
        memo_string = ""
        loop do
          block = @memo.read(memo_block_size)
          memo_string << block
          break if block.rstrip.size < memo_block_size
{'fraction_non_alphanumeric': 0.08670270270270271, 'fraction_numerical': 0.003027027027027027, 'mean_word_length': 2.3643636363636364, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3466413', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1494, 'n_words': 291}