<commit_before># encoding: UTF-8

module MH

  module Predictor

    class Insert < Base

      def initialize
        super(2)
      end

      def observe(separators, symbols)
        @_context.reset!
        symbols.each.with_index do |symbol, index|
          @_context << symbol
          self << separators[index]
        end
        @_context << 0
        self << separators[-1]
        @_context.size.times { @_context << (self << 0) }
      end

      def generate(symbols)
        separators = []
        @_context.reset!
        symbols.each do |symbol|
          @_context << symbol
          separators << random
        end<commit_msg>fixed a final bug in insert model<commit_after># encoding: UTF-8

module MH

  module Predictor

    class Insert < Base

      def initialize
        super(2)
      end

      def observe(separators, symbols)
        @_context.reset!
        symbols.each.with_index do |symbol, index|
          @_context << symbol
          self << separators[index]
        end
        @_context << 0
        self << separators[-1]
      end

      def generate(symbols)
        separators = []
        @_context.reset!
        symbols.each do |symbol|
          @_context << symbol
          separators << random
        end
{'fraction_non_alphanumeric': 0.09047619047619047, 'fraction_numerical': 0.007142857142857143, 'mean_word_length': 1.7002141327623126, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3808523', 'n_tokens_mistral': 366, 'n_tokens_neox': 350, 'n_words': 97}