<commit_before>require 'asciidoctor'
require 'colorize'

module Asciidoctor
  module DocTest
    module Generator

      ##
      # Generates missing, or rewrite existing output examples from the
      # input examples converted using the +renderer+.
      #
      # @param output_suite [BaseExamplesSuite] an instance of
      #        {BaseExamplesSuite} subclass to read and generate the output
      #        examples.
      #
      # @param input_suite [BaseExamplesSuite] an instance of
      #        {BaseExamplesSuite} subclass to read the reference input
      #        examples.
      #
      # @param renderer [#convert]
      #
      # @param pattern [String] glob-like pattern to select examples to
      #        (re)generate (see {BaseExample#name_match?}).
      #
      # @param rewrite [Boolean] whether to rewrite an already existing
      #        example.
      #
      # @param log_os [#<<] output stream where to write log messages.
      #
      def self.generate!(output_suite, input_suite, renderer, pattern: '*:*',
                         rewrite: false, log_os: $stdout)
        updated = []

        input_suite.pair_with(output_suite).each do |input, output|
          next unless input.name_match? pattern

          log = ->(msg, color = :default) do
            log_os << " --> #{(msg % input.name).colorize(color)}\n" if log_os
          end

          if input.empty?
            log["Unknown %s, doesn't exist in input examples!"]
          else
            rendered = output_suite.convert_example(input, output.opts, renderer)
            if output.empty?
              log['Generating %s', :magenta]
              updated << rendered
            elsif rendered == output
              log['Unchanged %s', :green]
            elsif rewrite
              log['Rewriting %s', :red]
              updated << rendered
            else
              log['Skipping %s', :yellow]
            end
          end
        end

        output_suite.update_examples updated unless updated.empty?
      end
    end
  end
end<commit_msg>Fix generator to not lose descriptions<commit_after>require 'asciidoctor'
require 'colorize'

module Asciidoctor
  module DocTest
    module Generator

      ##
      # Generates missing, or rewrite existing output examples from the
      # input examples converted using the +renderer+.
      #
      # @param output_suite [BaseExamplesSuite] an instance of
      #        {BaseExamplesSuite} subclass to read and generate the output
      #        examples.
      #
      # @param input_suite [BaseExamplesSuite] an instance of
      #        {BaseExamplesSuite} subclass to read the reference input
      #        examples.
      #
      # @param renderer [#convert]
      #
      # @param pattern [String] glob-like pattern to select examples to
      #        (re)generate (see {BaseExample#name_match?}).
      #
      # @param rewrite [Boolean] whether to rewrite an already existing
      #        example.
      #
      # @param log_os [#<<] output stream where to write log messages.
      #
      def self.generate!(output_suite, input_suite, renderer, pattern: '*:*',
                         rewrite: false, log_os: $stdout)
        updated = []

        input_suite.pair_with(output_suite).each do |input, output|
          next unless input.name_match? pattern

          log = ->(msg, color = :default) do
            log_os << " --> #{(msg % input.name).colorize(color)}\n" if log_os
          end

          if input.empty?
            log["Unknown %s, doesn't exist in input examples!"]
          else
            rendered = output_suite.convert_example(input, output.opts, renderer)
            rendered.desc = output.desc

            if output.empty?
              log['Generating %s', :magenta]
              updated << rendered
            elsif rendered == output
              log['Unchanged %s', :green]
            elsif rewrite
              log['Rewriting %s', :red]
              updated << rendered
            else
              log['Skipping %s', :yellow]
            end
          end
        end

        output_suite.update_examples updated unless updated.empty?
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.08954154727793696, 'fraction_numerical': 0.0, 'mean_word_length': 1.8419267299864315, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6421659', 'n_tokens_mistral': 1122, 'n_tokens_neox': 1080, 'n_words': 338}