<commit_before>module DBF
  class Record
    include Helpers
    
    attr_reader :attributes
    
    def initialize(table)
      @table, @data, @memo = table, table.data, table.memo
      initialize_values(table.columns)
      define_accessors
    end
    
    def ==(other)
      other.respond_to?(:attributes) && other.attributes == attributes
    end
    
    private
    
    def define_accessors
      @table.columns.each do |column|
        underscored_column_name = underscore(column.name)
        unless respond_to?(underscored_column_name)
          self.class.send :define_method, underscored_column_name do
            @attributes[column.name]
          end
        end
      end
    end
    
    def initialize_values(columns)
      @attributes = {}
      columns.each do |column|
        @attributes[column.name] = case column.type
        when 'N' # number
          column.decimal.zero? ? unpack_string(column).to_i : unpack_string(column).to_f
        when 'D' # date
          raw = unpack_string(column).strip
          unless raw.empty?
            parts = raw.match(DATE_REGEXP).captures.map {|n| n.to_i}
            begin
              Time.gm(*parts)
            rescue
              Date.new(*parts)
            end
          end
        when 'M' # memo
          starting_block = unpack_string(column).to_i
          read_memo(starting_block)
        when 'L' # logical
          unpack_string(column) =~ /^(y|t)$/i ? true : false
        when 'I' # integer
          unpack_integer(column)
        when 'T' # datetime
          unpack_datetime(column)
        else
          unpack_string(column).strip
        end
        @attributes[underscore(column.name)] = @attributes[column.name]
      end
    end
  
    def unpack_column(column)
      @data.read(column.length).to_s.unpack("a#{column.length}")
    end
  
    def unpack_string(column)
      unpack_column(column).to_s
    end
    
    def unpack_integer(column)
      @data.read(column.length).unpack("v").first
    end
    
    def unpack_datetime(column)
      days, milliseconds = @data.read(column.length).unpack('l2')
      hours = (milliseconds / MS_PER_HOUR).to_i
      minutes = ((milliseconds - (hours * MS_PER_HOUR)) / MS_PER_MINUTE).to_i
      seconds = ((milliseconds - (hours * MS_PER_HOUR) - (minutes * MS_PER_MINUTE)) / MS_PER_SECOND).to_i
      DateTime.jd(days, hours, minutes, seconds)
    end
  
    def read_memo(start_block)
      return nil if start_block < 1

      @table.memo_file_format == :fpt ? build_fpt_memo(start_block) : build_dbt_memo(start_block)
    end
    
    def build_fpt_memo(start_block)<commit_msg>Allow for missing memo files<commit_after>module DBF
  class Record
    include Helpers
    
    attr_reader :attributes
    
    def initialize(table)
      @table, @data, @memo = table, table.data, table.memo
      initialize_values(table.columns)
      define_accessors
    end
    
    def ==(other)
      other.respond_to?(:attributes) && other.attributes == attributes
    end
    
    private
    
    def define_accessors
      @table.columns.each do |column|
        underscored_column_name = underscore(column.name)
        unless respond_to?(underscored_column_name)
          self.class.send :define_method, underscored_column_name do
            @attributes[column.name]
          end
        end
      end
    end
    
    def initialize_values(columns)
      @attributes = {}
      columns.each do |column|
        @attributes[column.name] = case column.type
        when 'N' # number
          column.decimal.zero? ? unpack_string(column).to_i : unpack_string(column).to_f
        when 'D' # date
          raw = unpack_string(column).strip
          unless raw.empty?
            parts = raw.match(DATE_REGEXP).captures.map {|n| n.to_i}
            begin
              Time.gm(*parts)
            rescue
              Date.new(*parts)
            end
          end
        when 'M' # memo
          starting_block = unpack_string(column).to_i
          read_memo(starting_block)
        when 'L' # logical
          unpack_string(column) =~ /^(y|t)$/i ? true : false
        when 'I' # integer
          unpack_integer(column)
        when 'T' # datetime
          unpack_datetime(column)
        else
          unpack_string(column).strip
        end
        @attributes[underscore(column.name)] = @attributes[column.name]
        @attributes
      end
    end
  
    def unpack_column(column)
      @data.read(column.length).to_s.unpack("a#{column.length}")
    end
  
    def unpack_string(column)
      unpack_column(column).to_s
    end
    
    def unpack_integer(column)
      @data.read(column.length).unpack("v").first
    end
    
    def unpack_datetime(column)
      days, milliseconds = @data.read(column.length).unpack('l2')
      hours = (milliseconds / MS_PER_HOUR).to_i
      minutes = ((milliseconds - (hours * MS_PER_HOUR)) / MS_PER_MINUTE).to_i
      seconds = ((milliseconds - (hours * MS_PER_HOUR) - (minutes * MS_PER_MINUTE)) / MS_PER_SECOND).to_i
      DateTime.jd(days, hours, minutes, seconds)
    end
  
    def read_memo(start_block)
      return nil if !@table.has_memo_file? || start_block < 1

      @table.memo_file_format == :fpt ? build_fpt_memo(start_block) : build_dbt_memo(start_block)
    end
    
    def build_fpt_memo(start_block)
{'fraction_non_alphanumeric': 0.09952963311382879, 'fraction_numerical': 0.0007525870178739417, 'mean_word_length': 2.208207604103802, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5477685', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1673, 'n_words': 315}