<commit_before># frozen_string_literal: true

require 'multi_json'
require 'base64'
require 'cucumber/formatter/backtrace_filter'
require 'cucumber/formatter/io'
require 'cucumber/formatter/ast_lookup'

module Cucumber
  module Formatter
    # The formatter used for <tt>--format json</tt>
    class Json
      include Io

      def initialize(config)
        @io = ensure_io(config.out_stream)
        @ast_lookup = AstLookup.new(config)
        @feature_hashes = []
        @step_or_hook_hash = {}
        config.on_event :test_case_started, &method(:on_test_case_started)
        config.on_event :test_case_finished, &method(:on_test_case_finished)
        config.on_event :test_step_started, &method(:on_test_step_started)
        config.on_event :test_step_finished, &method(:on_test_step_finished)
        config.on_event :test_run_finished, &method(:on_test_run_finished)
      end

      def on_test_case_started(event)
        test_case = event.test_case
        builder = Builder.new(test_case, @ast_lookup)
        unless same_feature_as_previous_test_case?(test_case)
          @feature_hash = builder.feature_hash
          @feature_hashes << @feature_hash
        end
        @test_case_hash = builder.test_case_hash
        if builder.background?
          @in_background = true
          feature_elements << builder.background_hash
          @element_hash = builder.background_hash
        else
          @in_background = false
          feature_elements << @test_case_hash
          @element_hash = @test_case_hash
        end
        @any_step_failed = false
      end

      def on_test_step_started(event)
        test_step = event.test_step
        return if internal_hook?(test_step)
        if test_step.hook?
          @step_or_hook_hash = {}
          hooks_of_type(test_step) << @step_or_hook_hash
          return
        end
        if first_step_after_background?(test_step)
          @in_background = false
          feature_elements << @test_case_hash
          @element_hash = @test_case_hash
        end
        @step_or_hook_hash = create_step_hash(test_step)
        steps << @step_or_hook_hash
        @step_hash = @step_or_hook_hash
      end

      def on_test_step_finished(event)
        test_step, result = *event.attributes
        result = result.with_filtered_backtrace(Cucumber::Formatter::BacktraceFilter)
        return if internal_hook?(test_step)
        add_match_and_result(test_step, result)
        @any_step_failed = true if result.failed?
      end

      def on_test_case_finished(event)
        _test_case, result = *event.attributes
        result = result.with_filtered_backtrace(Cucumber::Formatter::BacktraceFilter)
        add_failed_around_hook(result) if result.failed? && !@any_step_failed
      end

      def on_test_run_finished(_event)
        @io.write(MultiJson.dump(@feature_hashes, pretty: true))
      end

      def puts(message)
        test_step_output << message
      end

      def embed(src, mime_type, _label)
        if File.file?(src)
          content = File.open(src, 'rb', &:read)
          data = encode64(content)
        elsif mime_type =~ /;base64$/
          mime_type = mime_type[0..-8]
          data = src
        else
          data = encode64(src)
        end
        test_step_embeddings << { mime_type: mime_type, data: data }
      end

      private

      def same_feature_as_previous_test_case?(test_case)
        current_feature[:uri] == test_case.location.file
      end

      def first_step_after_background?(test_step)
        @in_background && test_step.location.lines.max >= @test_case_hash[:line]
      end

      def internal_hook?(test_step)
        test_step.location.file.include?('lib/cucumber/')
      end

      def current_feature
        @feature_hash ||= {} # rubocop:disable Naming/MemoizedInstanceVariableName
      end

      def feature_elements
        @feature_hash[:elements] ||= []
      end

      def steps
        @element_hash[:steps] ||= []
      end

      def hooks_of_type(hook_step)
        case hook_step.text
        when 'Before hook'
          before_hooks
        when 'After hook'
          after_hooks
        when 'AfterStep hook'
          after_step_hooks
        else
          raise 'Unknown hook type ' + hook_step.to_s
        end
      end

      def before_hooks
        @element_hash[:before] ||= []
      end

      def after_hooks
        @element_hash[:after] ||= []
      end

      def around_hooks
        @element_hash[:around] ||= []
      end

      def after_step_hooks
        @step_hash[:after] ||= []
      end

      def test_step_output
        @step_or_hook_hash[:output] ||= []
      end

      def test_step_embeddings
        @step_or_hook_hash[:embeddings] ||= []
      end

      def create_step_hash(test_step)
        step_source = @ast_lookup.step_source(test_step).step
        step_hash = {
          keyword: step_source[:keyword],
          name: test_step.text,
          line: test_step.location.lines.min
        }
        step_hash[:doc_string] = create_doc_string_hash(step_source[:doc_string]) unless step_source[:doc_string].nil?
        step_hash[:rows] = create_data_table_value(step_source[:data_table]) unless step_source[:data_table].nil?
        step_hash
      end

      def create_doc_string_hash(doc_string)
        content_type = doc_string[:content_type] || ''
        {
          value: doc_string[:content],
          content_type: content_type,
          line: doc_string[:location][:line]
        }
      end

      def create_data_table_value(data_table)
        data_table[:rows].map do |row|
          { cells: row[:cells].map { |cell| cell[:value] } }
        end
      end

      def add_match_and_result(test_step, result)
        @step_or_hook_hash[:match] = create_match_hash(test_step, result)
        @step_or_hook_hash[:result] = create_result_hash(result)
      end
<commit_msg>Let Json formatter use embeddings listed in Result<commit_after># frozen_string_literal: true

require 'multi_json'
require 'base64'
require 'cucumber/formatter/backtrace_filter'
require 'cucumber/formatter/io'
require 'cucumber/formatter/ast_lookup'

module Cucumber
  module Formatter
    # The formatter used for <tt>--format json</tt>
    class Json
      include Io

      def initialize(config)
        @io = ensure_io(config.out_stream)
        @ast_lookup = AstLookup.new(config)
        @feature_hashes = []
        @step_or_hook_hash = {}
        config.on_event :test_case_started, &method(:on_test_case_started)
        config.on_event :test_case_finished, &method(:on_test_case_finished)
        config.on_event :test_step_started, &method(:on_test_step_started)
        config.on_event :test_step_finished, &method(:on_test_step_finished)
        config.on_event :test_run_finished, &method(:on_test_run_finished)
      end

      def on_test_case_started(event)
        test_case = event.test_case
        builder = Builder.new(test_case, @ast_lookup)
        unless same_feature_as_previous_test_case?(test_case)
          @feature_hash = builder.feature_hash
          @feature_hashes << @feature_hash
        end
        @test_case_hash = builder.test_case_hash
        if builder.background?
          @in_background = true
          feature_elements << builder.background_hash
          @element_hash = builder.background_hash
        else
          @in_background = false
          feature_elements << @test_case_hash
          @element_hash = @test_case_hash
        end
        @any_step_failed = false
      end

      def on_test_step_started(event)
        test_step = event.test_step
        return if internal_hook?(test_step)
        if test_step.hook?
          @step_or_hook_hash = {}
          hooks_of_type(test_step) << @step_or_hook_hash
          return
        end
        if first_step_after_background?(test_step)
          @in_background = false
          feature_elements << @test_case_hash
          @element_hash = @test_case_hash
        end
        @step_or_hook_hash = create_step_hash(test_step)
        steps << @step_or_hook_hash
        @step_hash = @step_or_hook_hash
      end

      def on_test_step_finished(event)
        test_step, result = *event.attributes
        result = result.with_filtered_backtrace(Cucumber::Formatter::BacktraceFilter)
        return if internal_hook?(test_step)
        add_match_and_result(test_step, result)
        @any_step_failed = true if result.failed?
      end

      def on_test_case_finished(event)
        _test_case, result = *event.attributes
        result = result.with_filtered_backtrace(Cucumber::Formatter::BacktraceFilter)
        add_failed_around_hook(result) if result.failed? && !@any_step_failed
      end

      def on_test_run_finished(_event)
        @io.write(MultiJson.dump(@feature_hashes, pretty: true))
      end

      def puts(message)
        test_step_output << message
      end

      def embed(src, mime_type, _label)
        if File.file?(src)
          content = File.open(src, 'rb', &:read)
          data = encode64(content)
        elsif mime_type =~ /;base64$/
          mime_type = mime_type[0..-8]
          data = src
        else
          data = encode64(src)
        end
        test_step_embeddings << { mime_type: mime_type, data: data }
      end

      private

      def same_feature_as_previous_test_case?(test_case)
        current_feature[:uri] == test_case.location.file
      end

      def first_step_after_background?(test_step)
        @in_background && test_step.location.lines.max >= @test_case_hash[:line]
      end

      def internal_hook?(test_step)
        test_step.location.file.include?('lib/cucumber/')
      end

      def current_feature
        @feature_hash ||= {} # rubocop:disable Naming/MemoizedInstanceVariableName
      end

      def feature_elements
        @feature_hash[:elements] ||= []
      end

      def steps
        @element_hash[:steps] ||= []
      end

      def hooks_of_type(hook_step)
        case hook_step.text
        when 'Before hook'
          before_hooks
        when 'After hook'
          after_hooks
        when 'AfterStep hook'
          after_step_hooks
        else
          raise 'Unknown hook type ' + hook_step.to_s
        end
      end

      def before_hooks
        @element_hash[:before] ||= []
      end

      def after_hooks
        @element_hash[:after] ||= []
      end

      def around_hooks
        @element_hash[:around] ||= []
      end

      def after_step_hooks
        @step_hash[:after] ||= []
      end

      def test_step_output
        @step_or_hook_hash[:output] ||= []
      end

      def test_step_embeddings
        @step_or_hook_hash[:embeddings] ||= []
      end

      def create_step_hash(test_step)
        step_source = @ast_lookup.step_source(test_step).step
        step_hash = {
          keyword: step_source[:keyword],
          name: test_step.text,
          line: test_step.location.lines.min
        }
        step_hash[:doc_string] = create_doc_string_hash(step_source[:doc_string]) unless step_source[:doc_string].nil?
        step_hash[:rows] = create_data_table_value(step_source[:data_table]) unless step_source[:data_table].nil?
        step_hash
      end

      def create_doc_string_hash(doc_string)
        content_type = doc_string[:content_type] || ''
        {
          value: doc_string[:content],
          content_type: content_type,
          line: doc_string[:location][:line]
        }
      end

      def create_data_table_value(data_table)
        data_table[:rows].map do |row|
          { cells: row[:cells].map { |cell| cell[:value] } }
        end
      end

      def add_match_and_result(test_step, result)
        @step_or_hook_hash[:match] = create_match_hash(test_step, result)
        @step_or_hook_hash[:result] = create_result_hash(result)
        result.embeddings.each { |e| embed(e['src'], e['mime_type'], e['label']) } if result.respond_to?(:embeddings)
      end

{'fraction_non_alphanumeric': 0.08947853017493931, 'fraction_numerical': 0.0016740604335816524, 'mean_word_length': 2.489485981308411, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2988895', 'n_tokens_mistral': 3731, 'n_tokens_neox': 3672, 'n_words': 644}