<commit_before>class GECOS
  class Dump
    
    WORDS_PER_LINE = 4
    FROZEN_CHARACTERS_PER_WORD = 5
    UNFROZEN_CHARACTERS_PER_WORD = 4
    
    def self.dump(words, options={})
      offset = options[:offset] || 0
      if options[:lines]
        limit = (options[:lines] * WORDS_PER_LINE - 1) + offset
        limit = words.size - 1 if limit >= words.size
      else 
        limit = words.size - 1
      end
      display_offset = (options[:display_offset] || offset) - offset
      stream = options[:to] || STDOUT
      decoder = Decoder.new
      decoder.words = words
      if options[:frozen]
        characters = decoder.frozen_characters
        character_block_size = FROZEN_CHARACTERS_PER_WORD
      else
        characters = decoder.characters
        character_block_size = UNFROZEN_CHARACTERS_PER_WORD
      end
      # TODO Refactor using Array#justify_rows
      address_width = ('%o'%(limit+display_offset)).size
      i = offset
      loop do
        break if i > limit
        j = [i+WORDS_PER_LINE-1, limit].min
        chunk = ((words[i..j].map{|w| '%012o'%w }) + ([' '*12] * WORDS_PER_LINE))[0,WORDS_PER_LINE]
        chars = characters[i*character_block_size, WORDS_PER_LINE*character_block_size]
        chars = (chars + ' '*(WORDS_PER_LINE*character_block_size))[0,WORDS_PER_LINE*character_block_size]
        if options[:escape]
          chars = chars.inspect[1..-2].scan(/\\\d{3}|\\[^\d\\]|\\\\|[^\\]/).map{|s| (s+'   ')[0,4]}.join
        else
          chars = chars.gsub(/[[:cntrl:]]/, '~')
        end
        address = '0' + ("%0#{address_width}o"%(i + display_offset))
        stream.puts "#{address} #{chunk.join(' ')} #{chars}"
        i += WORDS_PER_LINE
      end
    end
  end
end<commit_msg>Fix bug in gecos dump<commit_after>class GECOS
  class Dump
    
    WORDS_PER_LINE = 4
    FROZEN_CHARACTERS_PER_WORD = 5
    UNFROZEN_CHARACTERS_PER_WORD = 4
    
    def self.dump(words, options={})
      offset = options[:offset] || 0
      if options[:lines]
        limit = (options[:lines] * WORDS_PER_LINE - 1) + offset
        limit = words.size - 1 if limit >= words.size
      else 
        limit = words.size - 1
      end
      display_offset = (options[:display_offset] || offset) - offset
      stream = options[:to] || STDOUT
      decoder = Decoder.new
      decoder.words = words
      if options[:frozen]
        characters = decoder.frozen_characters
        character_block_size = FROZEN_CHARACTERS_PER_WORD
      else
        characters = decoder.characters
        character_block_size = UNFROZEN_CHARACTERS_PER_WORD
      end
      # TODO Refactor using Array#justify_rows
      address_width = ('%o'%(limit+display_offset)).size
      i = offset
      loop do
        break if i > limit
        j = [i+WORDS_PER_LINE-1, limit].min
        chunk = ((words[i..j].map{|w| '%012o'%w }) + ([' '*12] * WORDS_PER_LINE))[0,WORDS_PER_LINE]
        chars = characters[i*character_block_size, WORDS_PER_LINE*character_block_size].join
        chars = (chars + ' '*(WORDS_PER_LINE*character_block_size))[0,WORDS_PER_LINE*character_block_size]
        if options[:escape]
          chars = chars.inspect[1..-2].scan(/\\\d{3}|\\[^\d\\]|\\\\|[^\\]/).map{|s| (s+'   ')[0,4]}.join
        else
          chars = chars.gsub(/[[:cntrl:]]/, '~')
        end
        address = '0' + ("%0#{address_width}o"%(i + display_offset))
        stream.puts "#{address} #{chunk.join(' ')} #{chars}"
        i += WORDS_PER_LINE
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.14097042779213323, 'fraction_numerical': 0.012632787826586276, 'mean_word_length': 2.4770459081836327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695050', 'n_tokens_mistral': 1258, 'n_tokens_neox': 1193, 'n_words': 221}