<commit_before>module Landescape
  # Tokenize: "\e[1;2fHi" to ['\e[1;2f', 'H', 'i']]
  class Tokenizer
    class << self
      def start(source)
        new(source).tap(&:start)
      end
    end

    attr_reader :source, :result

    def initialize(source)
      @result = Queue.new
      @source =
        if source.respond_to?(:getc)
          source
        else
          StringIO.new(source.to_s)
        end

      at_exit do
        @source.close unless @source.closed?
      end
    end

    def start
      @tokenize_thread = Thread.fork {
        while char = read_char
          begin
            tokens =
              case char
              when nil  then Thread.exit # StringIO#getc # => nil (EOF)
              when /\e/ then escape char
              else char
              end

            Array(tokens).each do |token|
              result.push token
            end
          rescue => e
            Landescape.logger.error <<-ERR.lstrip
              #{e.class.name} #{e.message}
              #{e.backtrace.join($/)}
            ERR
          end
        end
      }
    end

    def stop<commit_msg>It's not effects 2nd line<commit_after>module Landescape
  # Tokenize: "\e[1;2fHi" to ['\e[1;2f', 'H', 'i']]
  class Tokenizer
    class << self
      def start(source)
        new(source).tap(&:start)
      end
    end

    attr_reader :source, :result

    def initialize(source)
      @result = Queue.new
      @source =
        if source.respond_to?(:getc)
          source
        else
          StringIO.new(source.to_s)
        end

      at_exit do
        @source.close unless @source.closed?
      end
    end

    def start
      @tokenize_thread = Thread.fork {
        while char = read_char
          begin
            tokens =
              case char
              when nil  then Thread.exit # StringIO#getc # => nil (EOF)
              when /\e/ then escape char
              else char
              end

            Array(tokens).each do |token|
              result.push token
            end
          rescue => e
            Landescape.logger.error <<-ERR
              #{e.class.name} #{e.message}
              #{e.backtrace.join($/)}
            ERR
          end
        end
      }
    end

    def stop
{'fraction_non_alphanumeric': 0.09675987572126055, 'fraction_numerical': 0.0039946737683089215, 'mean_word_length': 1.2906504065040652, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6689829', 'n_tokens_mistral': 670, 'n_tokens_neox': 655, 'n_words': 169}