<commit_before># encoding: UTF-8
module Earthquake
  module Output
    def output_filters
      @output_filters ||= []
    end

    def output_filter(&block)
      output_filters << block
    end

    def outputs
      @outputs ||= []
    end

    def output(name = nil, &block)
      if block
        outputs.delete_if { |o| o[:name] == name } if name
        outputs << {:name => name, :block => block}
      else
        return if item_queue.empty?
        insert do
          while item = item_queue.shift
            item["_stream"] = true
            puts_items(item)
          end
        end
      end
    end

    def puts_items(items)
      [items].flatten.reverse_each do |item|
        next if output_filters.any? { |f| f.call(item) == false }
        outputs.each do |o|
          begin
            o[:block].call(item)
          rescue => e
            error e
          end
        end
      end
    end

    def insert(*messages)
      clear_line
      puts messages unless messages.empty?
      yield if block_given?
    ensure
      Readline.refresh_line
    end

    def clear_line
      print "\e[0G" + "\e[K"
    end

    def color_of(screen_name)
      config[:colors][screen_name.delete("^0-9A-Za-z_").to_i(36) % config[:colors].size]
    end
  end

  init do
    outputs.clear
    output_filters.clear

    config[:colors] ||= (31..36).to_a + (91..96).to_a
    config[:color] ||= {}
    config[:color].reverse_merge!(
      :info   => 90,
      :notice => 31,
      :event  => 42,
      :url    => [4, 36]
    )
    config[:raw_text] ||= false

    output :tweet do |item|
      next unless item["text"]

      info = []
      if item["in_reply_to_status_id"]
        info << "(reply to #{id2var(item["in_reply_to_status_id"])})"
      elsif item["retweeted_status"]
        info << "(retweet of #{id2var(item["retweeted_status"]["id"])})"
      end<commit_msg>fix #insert to call Readline.refresh_line on demand<commit_after># encoding: UTF-8
require 'stringio'

module Earthquake
  module Output
    def output_filters
      @output_filters ||= []
    end

    def output_filter(&block)
      output_filters << block
    end

    def outputs
      @outputs ||= []
    end

    def output(name = nil, &block)
      if block
        outputs.delete_if { |o| o[:name] == name } if name
        outputs << {:name => name, :block => block}
      else
        return if item_queue.empty?
        insert do
          while item = item_queue.shift
            item["_stream"] = true
            puts_items(item)
          end
        end
      end
    end

    def puts_items(items)
      [items].flatten.reverse_each do |item|
        next if output_filters.any? { |f| f.call(item) == false }
        outputs.each do |o|
          begin
            o[:block].call(item)
          rescue => e
            error e
          end
        end
      end
    end

    def insert(*messages)
      $stdout = StringIO.new

      puts messages
      yield if block_given?

      unless $stdout.string.empty?
        STDOUT.print "\e[0G\e[K#{$stdout.string}"
        Readline.refresh_line
      end
    ensure
      $stdout = STDOUT
    end

    def color_of(screen_name)
      config[:colors][screen_name.delete("^0-9A-Za-z_").to_i(36) % config[:colors].size]
    end
  end

  init do
    outputs.clear
    output_filters.clear

    config[:colors] ||= (31..36).to_a + (91..96).to_a
    config[:color] ||= {}
    config[:color].reverse_merge!(
      :info   => 90,
      :notice => 31,
      :event  => 42,
      :url    => [4, 36]
    )
    config[:raw_text] ||= false

    output :tweet do |item|
      next unless item["text"]

      info = []
      if item["in_reply_to_status_id"]
        info << "(reply to #{id2var(item["in_reply_to_status_id"])})"
      elsif item["retweeted_status"]
        info << "(retweet of #{id2var(item["retweeted_status"]["id"])})"
      end
{'fraction_non_alphanumeric': 0.12725399278722307, 'fraction_numerical': 0.01287995878413189, 'mean_word_length': 1.9438968915845338, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1517767', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1249, 'n_words': 293}