<commit_before>class Parser
prechigh
  left '|'
preclow

rule
  target
    : docopt

  docopt
    : synopsis { result = val[0] }

  synopsis
    : synopses
        { result = val[0] }
    | synopses synopsis
        { result = @machine.new_node(:either, [val[0], val[2]]) }

  synopses
    : t_synopses_begin t_prog_name mutex
        { result = val[2] }
    | t_synopses_begin t_prog_name
        { result = @machine.new_node(:null, nil) }

  mutex
    : args
    | args '|' mutex
        { result = @machine.new_node(:either, [val[0], val[2]]) }

  args
    : arg
        { result = val[0] }
    | arg args
        { result = @machine.new_node(:cons, [val[0], val[1]]) }

  arg
    : arg_repeatable
    | arg_repeatable t_ldots
        { result = @machine.new_node(:one_or_more, val[0]) }

  arg_repeatable
    : t_arg
        { result = @machine.new_node(:arg, val[0]) }
    | t_var
        { result = @machine.new_node(:var, val[0]) }
    | arg_group
        { result = val[0] }
    | t_short_opt
        { result = @machine.new_node(:short_option, val[0]) }
    | t_short_opt_arged t_var
        { result = @machine.new_node(:short_option, [val[0], val[1]]) }
    | t_long_opt
        { result = @machine.new_node(:long_option, val[0]) }
    | t_long_opt_arged '=' t_var
        { result = @machine.new_node(:long_option, [val[0], "=", val[2]]) }
    | t_long_opt_arged t_var
        { result = @machine.new_node(:long_option, [val[0], val[1]]) }
    | t_options_shorthand
        { result = @machine.new_node(:options_shorthand, \
            @machine.short_options.collect do |x|
              @machine.new_node(:short_option, x)
            end)
        }


  arg_group
    : '(' mutex ')'
        { result = val[1] }
    | '[' mutex ']'
        { result = @machine.new_node(:optional, val[1])
          if val[1].kind_of? Docopt::Machine::Nodes::ShortOption then
            result = @machine.new_node(:options_shorthand, [val[1]])
          elsif val[1].kind_of? Docopt::Machine::Nodes::Cons \
            and val[1].cons_of? Docopt::Machine::Nodes::ShortOption
            then
            result = @machine.new_node(:options_shorthand, val[1].cons)
          end
        }

end

---- header

require File.expand_path("../../machine.rb", __FILE__)
require File.expand_path("../../options_block.rb", __FILE__)
require File.expand_path("../lexer.rb", __FILE__)

module Docopt
module UsageBlock
---- inner

  def initialize str
    @str = str
  end

  def parse
    @str += "\n"
    options = Docopt::parse_options(@str)
    @machine = Docopt::Machine.new(options)
    @lexer = Lexer.new(@str, @machine)
    do_parse
  end

  def next_token
    r = @lexer.next_token
    # puts r.to_s
    r
  end

  def on_error(error_token_id, error_value, value_stack)
    raise Docopt::LanguageError, "error \"%s\"" % \
      [error_value]
  end

---- footer
end
end
<commit_msg>cleanup parser for short opts with args<commit_after>class Parser
prechigh
  left '|'
preclow

rule
  target
    : docopt

  docopt
    : synopsis { result = val[0] }

  synopsis
    : synopses
        { result = val[0] }
    | synopses synopsis
        { result = @machine.new_node(:either, [val[0], val[2]]) }

  synopses
    : t_synopses_begin t_prog_name mutex
        { result = val[2] }
    | t_synopses_begin t_prog_name
        { result = @machine.new_node(:null, nil) }

  mutex
    : args
    | args '|' mutex
        { result = @machine.new_node(:either, [val[0], val[2]]) }

  args
    : arg
        { result = val[0] }
    | arg args
        { result = @machine.new_node(:cons, [val[0], val[1]]) }

  arg
    : arg_repeatable
    | arg_repeatable t_ldots
        { result = @machine.new_node(:one_or_more, val[0]) }

  arg_repeatable
    : t_arg
        { result = @machine.new_node(:arg, val[0]) }
    | t_var
        { result = @machine.new_node(:var, val[0]) }
    | arg_group
        { result = val[0] }
    | t_short_opt
        { result = @machine.new_node(:short_option, val[0]) }
    | t_short_opt_arged t_var
        { result = @machine.new_node(:short_option, val[0]) }
    | t_long_opt
        { result = @machine.new_node(:long_option, val[0]) }
    | t_long_opt_arged '=' t_var
        { result = @machine.new_node(:long_option, [val[0], "=", val[2]]) }
    | t_long_opt_arged t_var
        { result = @machine.new_node(:long_option, [val[0], val[1]]) }
    | t_options_shorthand
        { result = @machine.new_node(:options_shorthand, \
            @machine.short_options.collect do |x|
              @machine.new_node(:short_option, x)
            end)
        }


  arg_group
    : '(' mutex ')'
        { result = val[1] }
    | '[' mutex ']'
        { #HACK
          result = @machine.new_node(:optional, val[1])
          # if val[1].kind_of? Docopt::Machine::Nodes::ShortOption then
          #   result = @machine.new_node(:options_shorthand, [val[1]])
          # els
          if val[1].kind_of? Docopt::Machine::Nodes::Cons \
            and val[1].cons_of? Docopt::Machine::Nodes::ShortOption
            then
            result = @machine.new_node(:options_shorthand, val[1].cons)
          end
        }

end

---- header

require File.expand_path("../../machine.rb", __FILE__)
require File.expand_path("../../options_block.rb", __FILE__)
require File.expand_path("../lexer.rb", __FILE__)

module Docopt
module UsageBlock
---- inner

  def initialize str
    @str = str
  end

  def parse
    @str += "\n"
    options = Docopt::parse_options(@str)
    @machine = Docopt::Machine.new(options)
    @lexer = Lexer.new(@str, @machine)
    do_parse
  end

  def next_token
    r = @lexer.next_token
    # puts r.to_s
    r
  end

  def on_error(error_token_id, error_value, value_stack)
    raise Docopt::LanguageError, "error \"%s\"" % \
      [error_value]
  end

---- footer
end
end

{'fraction_non_alphanumeric': 0.14645506296360186, 'fraction_numerical': 0.009832672071761256, 'mean_word_length': 2.4005865102639294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7417227', 'n_tokens_mistral': 2101, 'n_tokens_neox': 2072, 'n_words': 381}