<commit_before>module Vertica
  module Messages
    class RowDescription < BackendMessage
      message_id 'T'

      attr_reader :fields

      def initialize(data)
        @fields = []
        field_count = data.unpack('n').first
        pos = 2
        field_count.times do |field_index|
          field_info = data.unpack("@#{pos}Z*NnNnNn")
          @fields << {
            :name             => field_info[0],
            :table_oid        => field_info[1],
            :attribute_number => field_info[2],
            :data_type_oid    => field_info[3],
            :data_type_size   => field_info[4],
            :type_modifier    => field_info[5],
            :format_code      => field_info[6],
          }
          
          pos += 19 + field_info[0].size
        end
      end
    end
  end
end
<commit_msg>add force_encoding to column name<commit_after>module Vertica
  module Messages
    class RowDescription < BackendMessage
      message_id 'T'

      attr_reader :fields

      def initialize(data)
        @fields = []
        field_count = data.unpack('n').first
        pos = 2
        field_count.times do |field_index|
          field_info = data.unpack("@#{pos}Z*NnNnNn")
          @fields << {
            :name             => field_info[0].force_encoding('UTF-8'),
            :table_oid        => field_info[1],
            :attribute_number => field_info[2],
            :data_type_oid    => field_info[3],
            :data_type_size   => field_info[4],
            :type_modifier    => field_info[5],
            :format_code      => field_info[6],
          }
          
          pos += 19 + field_info[0].size
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.10807600950118765, 'fraction_numerical': 0.013657957244655582, 'mean_word_length': 1.6535433070866141, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '527948', 'n_tokens_mistral': 521, 'n_tokens_neox': 512, 'n_words': 96}