<commit_before>require 'gherkin/ast_node'

module Gherkin
  class AstBuilder
    attr_reader :stack

    def initialize
      @stack = [AstNode.new(:None)]
    end

    def start_rule(rule_type)
      stack.push AstNode.new(rule_type)
    end

    def end_rule(rule_type)
      node = stack.pop
      current_node.add(node.rule_type, transform_node(node))
    end

    def build(token)
      current_node.add(token.matched_type, token)
    end

    def get_result
      current_node.get_single(:Feature)
    end

    def current_node
      stack.last
    end

    def get_location(token, column=nil)
      # TODO: translated from JS... is it right?
      (column.nil? || column.zero?) ? token.location : {line: token.location.line, column: column}
    end

    def get_tags(node)
      tags = []
      tags_node = node.get_single(:Tags)
      return tags unless tags_node

      tags_node.get_tokens(:TagLine).each do |token|
        token.matched_items.each do |tag_item|
          tags.push({
            location: get_location(token, tag_item.column),
            name: tag_item.text
          })
        end
      end

      tags
    end

    def get_table_rows(node)
      rows = node.get_tokens(:TableRow).map do |token|
        {
          location: get_location(token),
          cells: get_cells(token)
        }
      end
      #EnsureCellCount(rows);
      rows
    end

    def get_cells(table_row_token)
      table_row_token.matched_items.map do |cell_item|
        {
          location: get_location(table_row_token, cell_item.column),
          value: cell_item.text
        }
      end
    end

    def get_description(node)
      node.get_single(:Description)
    end

    def get_steps(node)
      node.get_items(:Step)
    end

    def transform_node(node)
      case node.rule_type
      when :Step
        step_line = node.get_token(:StepLine)
        step_argument = node.get_single(:DataTable) || node.get_single(:DocString) || nil

        reject_nils(
          type: node.rule_type,
          location: get_location(step_line),
          keyword: step_line.matched_keyword,
          name: step_line.matched_text,
          argument: step_argument
        )
      when :DocString
        separator_token = node.get_tokens(:DocStringSeparator)[0]
        content_type = separator_token.matched_text
        line_tokens = node.get_tokens(:Other)
        content = line_tokens.map { |t| t.matched_text }.join("\n")

        reject_nils(
          type: node.rule_type,
          location: get_location(separator_token),
          contentType: content_type,
          content: content
        )
      when :DataTable
        rows = get_table_rows(node)
        reject_nils(
          type: node.rule_type,
          rows: rows,
        )
      when :Background
        background_line = node.get_token(:BackgroundLine)
        description = get_description(node)
        steps = get_steps(node)

        reject_nils(
          type: node.rule_type,
          location: get_location(background_line),
          keyword: background_line.matched_keyword,
          name: background_line.matched_text,
          description: description,
          steps: steps
        )
      when :Scenario_Definition
        tags = get_tags(node)
        scenario_node = node.get_single(:Scenario)
        if(scenario_node)
          scenario_line = scenario_node.get_token(:ScenarioLine)
          description = get_description(scenario_node)
          steps = get_steps(scenario_node)

          reject_nils(
            type: scenario_node.rule_type,
            tags: tags,
            location: get_location(scenario_line),
            keyword: scenario_line.matched_keyword,
            name: scenario_line.matched_text,
            description: description,
            steps: steps
          )
        else
          scenario_outline_node = node.get_single(:ScenarioOutline)
          raise 'Internal grammar error' unless scenario_outline_node

          scenario_outline_line = scenario_outline_node.get_token(:ScenarioOutlineLine)
          description = get_description(scenario_outline_node)
          steps = get_steps(scenario_outline_node)
          examples = scenario_outline_node.get_items(:Examples)

          reject_nils(
            type: scenario_outline_node.rule_type,
            tags: tags,
            location: get_location(scenario_outline_line),
            keyword: scenario_outline_line.matched_keyword,
            name: scenario_outline_line.matched_text,
            description: description,
            steps: steps,
            examples: examples
          )
        end
      when :Examples
        tags = get_tags(node)
        examples_line = node.get_token(:ExamplesLine)
        description = get_description(node)

        all_rows = get_table_rows(node)
        header = all_rows[0]
        rows = all_rows.slice(1)

        reject_nils(
          type: node.rule_type,
          tags: tags,
          location: get_location(examples_line),
          keyword: examples_line.matched_keyword,
          name: examples_line.matched_text,
          description: description,
          header: header,
          rows: rows
        )
      when :Description
        line_tokens = node.get_tokens(:Other)
        # Trim trailing empty lines
        last_non_empty = line_tokens.rindex { |token| !token.line.trimmed_line_text.empty? }
        description = line_tokens[0..last_non_empty].map { |token| token.matched_text }.join("\n")
        return description
      when :Feature
        header = node.get_single(:Feature_Header)
        tags = get_tags(header)
        feature_line = header.get_token(:FeatureLine)
        background = node.get_single(:Background)
        scenario_definitions = node.get_items(:Scenario_Definition)
        description = get_description(header)
        language = feature_line.matched_gherkin_dialect

        reject_nils(
          type: node.rule_type,
          tags: tags,
          location: get_location(feature_line),
          language: language,
          keyword: feature_line.matched_keyword,
          name: feature_line.matched_text,
          description: description,
          background: background,
          scenarioDefinitions: scenario_definitions
        )
      else
        return node
      end
    end

    def reject_nils(values)
      values.reject { |k,v| v.nil? }
    end
  end
end
<commit_msg>Split example table into header and rows correctly<commit_after>require 'gherkin/ast_node'

module Gherkin
  class AstBuilder
    attr_reader :stack

    def initialize
      @stack = [AstNode.new(:None)]
    end

    def start_rule(rule_type)
      stack.push AstNode.new(rule_type)
    end

    def end_rule(rule_type)
      node = stack.pop
      current_node.add(node.rule_type, transform_node(node))
    end

    def build(token)
      current_node.add(token.matched_type, token)
    end

    def get_result
      current_node.get_single(:Feature)
    end

    def current_node
      stack.last
    end

    def get_location(token, column=nil)
      # TODO: translated from JS... is it right?
      (column.nil? || column.zero?) ? token.location : {line: token.location.line, column: column}
    end

    def get_tags(node)
      tags = []
      tags_node = node.get_single(:Tags)
      return tags unless tags_node

      tags_node.get_tokens(:TagLine).each do |token|
        token.matched_items.each do |tag_item|
          tags.push({
            location: get_location(token, tag_item.column),
            name: tag_item.text
          })
        end
      end

      tags
    end

    def get_table_rows(node)
      rows = node.get_tokens(:TableRow).map do |token|
        {
          location: get_location(token),
          cells: get_cells(token)
        }
      end
      #EnsureCellCount(rows);
      rows
    end

    def get_cells(table_row_token)
      table_row_token.matched_items.map do |cell_item|
        {
          location: get_location(table_row_token, cell_item.column),
          value: cell_item.text
        }
      end
    end

    def get_description(node)
      node.get_single(:Description)
    end

    def get_steps(node)
      node.get_items(:Step)
    end

    def transform_node(node)
      case node.rule_type
      when :Step
        step_line = node.get_token(:StepLine)
        step_argument = node.get_single(:DataTable) || node.get_single(:DocString) || nil

        reject_nils(
          type: node.rule_type,
          location: get_location(step_line),
          keyword: step_line.matched_keyword,
          name: step_line.matched_text,
          argument: step_argument
        )
      when :DocString
        separator_token = node.get_tokens(:DocStringSeparator)[0]
        content_type = separator_token.matched_text
        line_tokens = node.get_tokens(:Other)
        content = line_tokens.map { |t| t.matched_text }.join("\n")

        reject_nils(
          type: node.rule_type,
          location: get_location(separator_token),
          contentType: content_type,
          content: content
        )
      when :DataTable
        rows = get_table_rows(node)
        reject_nils(
          type: node.rule_type,
          rows: rows,
        )
      when :Background
        background_line = node.get_token(:BackgroundLine)
        description = get_description(node)
        steps = get_steps(node)

        reject_nils(
          type: node.rule_type,
          location: get_location(background_line),
          keyword: background_line.matched_keyword,
          name: background_line.matched_text,
          description: description,
          steps: steps
        )
      when :Scenario_Definition
        tags = get_tags(node)
        scenario_node = node.get_single(:Scenario)
        if(scenario_node)
          scenario_line = scenario_node.get_token(:ScenarioLine)
          description = get_description(scenario_node)
          steps = get_steps(scenario_node)

          reject_nils(
            type: scenario_node.rule_type,
            tags: tags,
            location: get_location(scenario_line),
            keyword: scenario_line.matched_keyword,
            name: scenario_line.matched_text,
            description: description,
            steps: steps
          )
        else
          scenario_outline_node = node.get_single(:ScenarioOutline)
          raise 'Internal grammar error' unless scenario_outline_node

          scenario_outline_line = scenario_outline_node.get_token(:ScenarioOutlineLine)
          description = get_description(scenario_outline_node)
          steps = get_steps(scenario_outline_node)
          examples = scenario_outline_node.get_items(:Examples)

          reject_nils(
            type: scenario_outline_node.rule_type,
            tags: tags,
            location: get_location(scenario_outline_line),
            keyword: scenario_outline_line.matched_keyword,
            name: scenario_outline_line.matched_text,
            description: description,
            steps: steps,
            examples: examples
          )
        end
      when :Examples
        tags = get_tags(node)
        examples_line = node.get_token(:ExamplesLine)
        description = get_description(node)

        all_rows = get_table_rows(node)
        header, *rows = all_rows

        reject_nils(
          type: node.rule_type,
          tags: tags,
          location: get_location(examples_line),
          keyword: examples_line.matched_keyword,
          name: examples_line.matched_text,
          description: description,
          header: header,
          rows: rows
        )
      when :Description
        line_tokens = node.get_tokens(:Other)
        # Trim trailing empty lines
        last_non_empty = line_tokens.rindex { |token| !token.line.trimmed_line_text.empty? }
        description = line_tokens[0..last_non_empty].map { |token| token.matched_text }.join("\n")
        return description
      when :Feature
        header = node.get_single(:Feature_Header)
        tags = get_tags(header)
        feature_line = header.get_token(:FeatureLine)
        background = node.get_single(:Background)
        scenario_definitions = node.get_items(:Scenario_Definition)
        description = get_description(header)
        language = feature_line.matched_gherkin_dialect

        reject_nils(
          type: node.rule_type,
          tags: tags,
          location: get_location(feature_line),
          language: language,
          keyword: feature_line.matched_keyword,
          name: feature_line.matched_text,
          description: description,
          background: background,
          scenarioDefinitions: scenario_definitions
        )
      else
        return node
      end
    end

    def reject_nils(values)
      values.reject { |k,v| v.nil? }
    end
  end
end

{'fraction_non_alphanumeric': 0.0784956304619226, 'fraction_numerical': 0.00046816479400749064, 'mean_word_length': 2.179608037707765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5599385', 'n_tokens_mistral': 3692, 'n_tokens_neox': 3631, 'n_words': 708}