<commit_before>module CC
  module Analyzer
    module Formatters
      class JSONFormatter < Formatter

        def engine_running(engine)
          @active_engine = engine
          yield
          @active_engine = nil
        end

        def started
          puts "[ "
        end

        def finished
          puts "\b\b ]"
        end

        def write(data)
          return unless data.present?

          document = JSON.parse(data)
          document["engine_name"] = @active_engine.name
          puts document.to_json + ","
        end

        def failed(output)
          $stderr.puts "\nAnalysis failed with the following output:"
          $stderr.puts output
          exit 1
        end
      end
    end
  end
end<commit_msg>update formatting to be aware of started state<commit_after>module CC
  module Analyzer
    module Formatters
      class JSONFormatter < Formatter

        def initialize
          @has_begun = false
        end

        def engine_running(engine)
          @active_engine = engine
          yield
          @active_engine = nil
        end

        def started
          print "[ "
        end

        def finished
          print " ]\n"
        end

        def write(data)
          return unless data.present?

          document = JSON.parse(data)
          document["engine_name"] = @active_engine.name

          if @has_begun
            print ",\n" + document.to_json
          else
            print document.to_json
            @has_begun = true
          end
        end

        def failed(output)
          $stderr.puts "\nAnalysis failed with the following output:"
          $stderr.puts output
          exit 1
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.05944055944055944, 'fraction_numerical': 0.0011655011655011655, 'mean_word_length': 1.3980446927374302, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2130310', 'n_tokens_mistral': 457, 'n_tokens_neox': 436, 'n_words': 132}