<commit_before>require 'cucumber/messages'
require 'gherkin/parser'
require 'gherkin/token_matcher'
require 'gherkin/pickles/compiler'

module Gherkin
  module Stream
    class ParserMessageStream
      def initialize(paths, sources, options)
        @paths = paths
        @sources = sources
        @options = options

        id_generator = options[:id_generator] || Cucumber::Messages::IdGenerator::UUID.new
        @parser = Parser.new(AstBuilder.new(id_generator))
        @compiler = Pickles::Compiler.new(id_generator)
      end

      def messages
        enumerated = false
        Enumerator.new do |y|
          raise DoubleIterationException, "Messages have already been enumerated" if enumerated
          enumerated = true

          sources.each do |source|
            y.yield(Cucumber::Messages::Envelope.new(source: source)) if @options[:include_source]
            begin
              gherkin_document = nil

              if @options[:include_gherkin_document]
                gherkin_document = build_gherkin_document(source)
                y.yield(Cucumber::Messages::Envelope.new(gherkin_document: gherkin_document))
              end
              if @options[:include_pickles]
                gherkin_document ||= build_gherkin_document(source)
                pickles = @compiler.compile(gherkin_document, source)
                pickles.each do |pickle|
                  y.yield(Cucumber::Messages::Envelope.new(pickle: pickle))
                end
              end
            rescue CompositeParserException => err
              yield_error_attachments(y, err.errors, source.uri)
            rescue ParserException => err
              yield_error_attachments(y, [err], source.uri)
            end
          end
        end
      end

      private

      def yield_error_attachments(y, errors, uri)
        errors.each do |err|
          attachment = Cucumber::Messages::Attachment.new(
            source: Cucumber::Messages::SourceReference.new(
              uri: uri,
              location: Cucumber::Messages::Location.new(
                line: err.location[:line],
                column: err.location[:column]
              )
            ),
            body: err.message
          )
          y.yield(Cucumber::Messages::Envelope.new(attachment: attachment))
        end
      end

      def sources
        Enumerator.new do |y|
          @paths.each do |path|
            source = Cucumber::Messages::Source.new(
              uri: path,
              data: File.open(path, 'r:UTF-8', &:read),
              media_type: 'text/x.cucumber.gherkin+plain'
            )
            y.yield(source)
          end
          @sources.each do |source|
            y.yield(source)
          end
        end
      end

      def build_gherkin_document(source)
        if @options[:default_dialect]
          token_matcher = TokenMatcher.new(@options[:default_dialect])
          gd = @parser.parse(source.data, token_matcher)
        else
          gd = @parser.parse(source.data)
        end
        gd[:uri] = source.uri
        Cucumber::Messages::GherkinDocument.new(gd)
      end
    end
  end
end
<commit_msg>Use ParseError instead of Attachment. Closes #936<commit_after>require 'cucumber/messages'
require 'gherkin/parser'
require 'gherkin/token_matcher'
require 'gherkin/pickles/compiler'

module Gherkin
  module Stream
    class ParserMessageStream
      def initialize(paths, sources, options)
        @paths = paths
        @sources = sources
        @options = options

        id_generator = options[:id_generator] || Cucumber::Messages::IdGenerator::UUID.new
        @parser = Parser.new(AstBuilder.new(id_generator))
        @compiler = Pickles::Compiler.new(id_generator)
      end

      def messages
        enumerated = false
        Enumerator.new do |y|
          raise DoubleIterationException, "Messages have already been enumerated" if enumerated
          enumerated = true

          sources.each do |source|
            y.yield(Cucumber::Messages::Envelope.new(source: source)) if @options[:include_source]
            begin
              gherkin_document = nil

              if @options[:include_gherkin_document]
                gherkin_document = build_gherkin_document(source)
                y.yield(Cucumber::Messages::Envelope.new(gherkin_document: gherkin_document))
              end
              if @options[:include_pickles]
                gherkin_document ||= build_gherkin_document(source)
                pickles = @compiler.compile(gherkin_document, source)
                pickles.each do |pickle|
                  y.yield(Cucumber::Messages::Envelope.new(pickle: pickle))
                end
              end
            rescue CompositeParserException => err
              yield_parse_errors(y, err.errors, source.uri)
            rescue ParserException => err
              yield_parse_errors(y, [err], source.uri)
            end
          end
        end
      end

      private

      def yield_parse_errors(y, errors, uri)
        errors.each do |err|
          parse_error = Cucumber::Messages::Attachment.new(
            source: Cucumber::Messages::SourceReference.new(
              uri: uri,
              location: Cucumber::Messages::Location.new(
                line: err.location[:line],
                column: err.location[:column]
              )
            ),
            message: err.message
          )
          y.yield(Cucumber::Messages::Envelope.new(parse_error: parse_error))
        end
      end

      def sources
        Enumerator.new do |y|
          @paths.each do |path|
            source = Cucumber::Messages::Source.new(
              uri: path,
              data: File.open(path, 'r:UTF-8', &:read),
              media_type: 'text/x.cucumber.gherkin+plain'
            )
            y.yield(source)
          end
          @sources.each do |source|
            y.yield(source)
          end
        end
      end

      def build_gherkin_document(source)
        if @options[:default_dialect]
          token_matcher = TokenMatcher.new(@options[:default_dialect])
          gd = @parser.parse(source.data, token_matcher)
        else
          gd = @parser.parse(source.data)
        end
        gd[:uri] = source.uri
        Cucumber::Messages::GherkinDocument.new(gd)
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.09167061798640747, 'fraction_numerical': 0.000790263948158685, 'mean_word_length': 1.9000916590284143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3006715', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1741, 'n_words': 346}