<commit_before>require 'bert/errno'

module FFI
  module BERT
    class Encoder < FFI::ManagedStruct

      include Errno

      def initialize(ptr)
        @ptr = ptr
      end

      def self.release(ptr)
        BERT.bert_encoder_destroy(ptr)
      end

      def Encoder.create
        unless (ptr = BERT.bert_encoder_create())
          Errno.raise_error(Errno::MALLOC)
        end

        return Encoder.new(ptr)
      end

      def Encoder.buffer(length,&block)
        encoder = Encoder.create
        buffer = FFI::Buffer.new(length)

        BERT.bert_encoder_buffer(encoder,buffer)

        block.call(encoder) if block
        return buffer
      end

      def Encoder.stream(io,&block)
        callback_proc = Proc.new { |data,length,user_data|
          io.write(data.get_string(0,length))
        }

        Encoder.callback(callback_proc,&block)
      end

      def Encoder.callback(callback_proc,&block)
        encoder = Encoder.create

        BERT.bert_encoder_callback(encoder,callback_proc,nil)

        block.call(encoder) if block
        return nil
      end

      def push(data)
        catch_error BERT.bert_encoder_push(self,data)
        return self
      end

      def <<(data)
        data = Data.from_ruby(data)

        push(data)

        data.destroy!
        return self
      end

      def to_ptr
        @ptr
      end

    end
  end
end<commit_msg>Made BERT.create protected.<commit_after>require 'bert/errno'

module FFI
  module BERT
    class Encoder < FFI::ManagedStruct

      include Errno

      def initialize(ptr)
        @ptr = ptr
      end

      def self.release(ptr)
        BERT.bert_encoder_destroy(ptr)
      end

      def Encoder.buffer(length,&block)
        encoder = Encoder.create
        buffer = FFI::Buffer.new(length)

        BERT.bert_encoder_buffer(encoder,buffer)

        block.call(encoder) if block
        return buffer
      end

      def Encoder.stream(io,&block)
        callback_proc = Proc.new { |data,length,user_data|
          io.write(data.get_string(0,length))
        }

        Encoder.callback(callback_proc,&block)
      end

      def Encoder.callback(callback_proc,&block)
        encoder = Encoder.create

        BERT.bert_encoder_callback(encoder,callback_proc,nil)

        block.call(encoder) if block
        return nil
      end

      def push(data)
        catch_error BERT.bert_encoder_push(self,data)
        return self
      end

      def <<(data)
        data = Data.from_ruby(data)

        push(data)

        data.destroy!
        return self
      end

      def to_ptr
        @ptr
      end

      protected

      def Encoder.create
        unless (ptr = BERT.bert_encoder_create())
          Errno.raise_error(Errno::MALLOC)
        end

        return Encoder.new(ptr)
      end

    end
  end
end
{'fraction_non_alphanumeric': 0.08087974459028024, 'fraction_numerical': 0.000709471443774388, 'mean_word_length': 1.922279792746114, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5244191', 'n_tokens_mistral': 885, 'n_tokens_neox': 830, 'n_words': 172}