<commit_before># encoding: utf-8

module Humboldt
  module TypeConverter
    class Binary
      HADOOP = ::Hadoop::Io::BytesWritable
      RUBY = ::String

      attr_reader :hadoop

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
        @hadoop = value
      end

      def initialize
        @hadoop = HADOOP.new
      end

      def ruby
        String.from_java_bytes(@hadoop.bytes).byteslice(0, @hadoop.length)
      end

      def ruby=(value)
        unless value.is_a?(RUBY)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end

        @hadoop.set(value.to_java_bytes, 0, value.bytesize)
      end
    end

    begin
      require 'msgpack'

      class Encoded < Binary
        def ruby=(value)
          unless value.is_a?(Hash) || value.is_a?(Array)
            raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected Hash or Array"
          end
          packed = MessagePack.pack(value)
          @hadoop.set(packed.to_java_bytes, 0, packed.bytesize)
        end

        def ruby
          packed = String.from_java_bytes(@hadoop.bytes).byteslice(0, @hadoop.length)
          MessagePack.unpack(packed, encoding: Encoding::UTF_8)
        end
      end
    rescue LoadError
    end

    class Text
      HADOOP = ::Hadoop::Io::Text
      RUBY = ::String

      attr_reader :hadoop

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
        @hadoop = value
      end

      def initialize
        @hadoop = HADOOP.new
      end

      def ruby
        String.from_java_bytes(@hadoop.bytes).byteslice(0, @hadoop.length).force_encoding(Encoding::UTF_8)
      end

      def ruby=(value)
        unless value.is_a?(RUBY)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end

        if value.encoding == Encoding::UTF_8
          @hadoop.set(value.to_java_bytes, 0, value.bytesize)
        else
          @hadoop.set(value)
        end
      end
    end

    begin
      require 'json'

      class Json < Text
        def ruby=(value)
          unless value.is_a?(Hash) || value.is_a?(Array)
            raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected Hash or Array"
          end
          @hadoop.set(JSON.generate(value))
        end

        def ruby
          JSON.parse(hadoop.to_s)
        end
      end
    end

    class Long
      HADOOP = ::Hadoop::Io::LongWritable
      RUBY = ::Integer

      attr_reader :hadoop

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
        @hadoop = value
      end

      def initialize
        @hadoop = HADOOP.new
      end

      def ruby
        @hadoop.get
      end

      def ruby=(value)
        unless value.is_a?(Integer)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end

        @hadoop.set value
      end
    end

    class None
      HADOOP = ::Hadoop::Io::NullWritable
      RUBY = ::NilClass

      def hadoop
        HADOOP.get
      end

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
      end

      def ruby
        nil
      end

      def ruby=(value)
        unless value.nil?
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end
      end
    end

    TYPE_CONVERTER_CLASS_CACHE = Hash.new { |h,k| h[k] = const_get(k.to_s.capitalize) }

    def self.[](name)
      TYPE_CONVERTER_CLASS_CACHE[name]
    end

    FROM_HADOOP_MAPPINGS = {
      ::Hadoop::Io::Text => Text,
      ::Hadoop::Io::BytesWritable => Binary,
      ::Hadoop::Io::LongWritable => Long,
      ::Hadoop::Io::NullWritable => None
    }.freeze

    def self.from_hadoop(hadoop_class)
      accessor = FROM_HADOOP_MAPPINGS[hadoop_class]
      raise ArgumentError, "Unsupported Hadoop type: #{hadoop_class}" unless accessor
      accessor
    end
  end
end
<commit_msg>Reimplement Json/Message pack converters to call super<commit_after># encoding: utf-8

module Humboldt
  module TypeConverter
    class Binary
      HADOOP = ::Hadoop::Io::BytesWritable
      RUBY = ::String

      attr_reader :hadoop

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
        @hadoop = value
      end

      def initialize
        @hadoop = HADOOP.new
      end

      def ruby
        String.from_java_bytes(@hadoop.bytes).byteslice(0, @hadoop.length)
      end

      def ruby=(value)
        unless value.is_a?(RUBY)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end

        @hadoop.set(value.to_java_bytes, 0, value.bytesize)
      end
    end

    begin
      require 'msgpack'

      class Encoded < Binary
        def ruby=(value)
          unless value.is_a?(Hash) || value.is_a?(Array)
            raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected Hash or Array"
          end
          super(MessagePack.pack(value))
        end

        def ruby
          MessagePack.unpack(super(), encoding: Encoding::UTF_8)
        end
      end
    rescue LoadError
    end

    class Text
      HADOOP = ::Hadoop::Io::Text
      RUBY = ::String

      attr_reader :hadoop

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
        @hadoop = value
      end

      def initialize
        @hadoop = HADOOP.new
      end

      def ruby
        String.from_java_bytes(@hadoop.bytes).byteslice(0, @hadoop.length).force_encoding(Encoding::UTF_8)
      end

      def ruby=(value)
        unless value.is_a?(RUBY)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end

        if value.encoding == Encoding::UTF_8
          @hadoop.set(value.to_java_bytes, 0, value.bytesize)
        else
          @hadoop.set(value)
        end
      end
    end

    begin
      require 'json'

      class Json < Text
        def ruby=(value)
          unless value.is_a?(Hash) || value.is_a?(Array)
            raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected Hash or Array"
          end
          super(JSON.generate(value))
        end

        def ruby
          JSON.parse(super())
        end
      end
    end

    class Long
      HADOOP = ::Hadoop::Io::LongWritable
      RUBY = ::Integer

      attr_reader :hadoop

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
        @hadoop = value
      end

      def initialize
        @hadoop = HADOOP.new
      end

      def ruby
        @hadoop.get
      end

      def ruby=(value)
        unless value.is_a?(Integer)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end

        @hadoop.set value
      end
    end

    class None
      HADOOP = ::Hadoop::Io::NullWritable
      RUBY = ::NilClass

      def hadoop
        HADOOP.get
      end

      def hadoop=(value)
        unless value.is_a?(HADOOP)
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{HADOOP}"
        end
      end

      def ruby
        nil
      end

      def ruby=(value)
        unless value.nil?
          raise ArgumentError, "Hadoop type mismatch, was #{value.class}, expected #{RUBY}"
        end
      end
    end

    TYPE_CONVERTER_CLASS_CACHE = Hash.new { |h,k| h[k] = const_get(k.to_s.capitalize) }

    def self.[](name)
      TYPE_CONVERTER_CLASS_CACHE[name]
    end

    FROM_HADOOP_MAPPINGS = {
      ::Hadoop::Io::Text => Text,
      ::Hadoop::Io::BytesWritable => Binary,
      ::Hadoop::Io::LongWritable => Long,
      ::Hadoop::Io::NullWritable => None
    }.freeze

    def self.from_hadoop(hadoop_class)
      accessor = FROM_HADOOP_MAPPINGS[hadoop_class]
      raise ArgumentError, "Unsupported Hadoop type: #{hadoop_class}" unless accessor
      accessor
    end
  end
end

{'fraction_non_alphanumeric': 0.09992987377279103, 'fraction_numerical': 0.0021037868162692847, 'mean_word_length': 2.0258132956152757, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5876792', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2597, 'n_words': 651}