<commit_before>.HLL '_Tcl', ''
.namespace

=head1 NAME

Tcl Parser

=head1 DESCRIPTION

This is the parser that makes up the heart of Partcl. It follows
the 11 rules that are found in the Tcl manpage, available online
at <http://www.tcl.tk/man/tcl8.4/TclCmd/Tcl.htm>.

=head1 FUNCTIONS

=over 4

=item C<pmc commands = parse(string tcl_code)>

Parses the Tcl code and returns an array of TclCommand objects.
First, it performs the \<newline> substitution. Then it fetches
commands, one at a time (skipping over comments).

=cut

.sub parse
  .param string tcl_code

  .local int len
  len = length tcl_code
 
  # perform the backslash-newline substitution
  $I0 = -1
loop:
  inc $I0
  if $I0 >= len goto get_commands
  $I1 = ord tcl_code, $I0
  if $I1 != 92 goto loop # \\
  inc $I0
  $I2 = $I0
  $I1 = ord tcl_code, $I2
  if $I1 != 10 goto loop # \n
space:
  inc $I2
  if $I0 >= len goto get_commands
  $I1 = is_cclass .CCLASS_WHITESPACE, tcl_code, $I2
  if $I1 == 0 goto not_space
  goto space
not_space:
  dec $I0
  $I1 = $I2 - $I0
  substr tcl_code, $I0, $I1, " "
  dec $I1
  len -= $I1
  goto loop

get_commands:
  # an array of commands
  .local pmc commands
  $I0 = find_type "TclCommandList"
  commands = new $I0
  
  # position in the file
  .local int pos
  pos = 0
  # characters that end commands
  .local pmc chars
  chars = new .Hash
  chars[10] = 1 # \n
  chars[59] = 1 # ;
  
next_command:
  # Do we have a comment? If so, skip to the next position where
  # We might have a command.
  .local int check_pos
  check_pos = skip_comment(tcl_code, pos)
  if check_pos == pos goto done_comment

found_comment:
  pos = check_pos
  goto next_command

done_comment:
  .local pmc command
  (command, pos) = get_command(tcl_code, chars, pos)
  if_null command, done
  
  push commands, command
  goto next_command

done:
  .return(commands)
.end

=item C<int pos = skip_comment(string tcl_code, int pos)>

Checks for a comment and returns either the original pos
or the position after the comment.

    Incoming: # comment\n
              ^
    Outgoing: # comment\n
                       ^^

=cut

.sub skip_comment
  .param string tcl_code
  .param int    pos

  .local pmc chars
  chars = new .Hash
  chars[10] = 1 # \n
  
  .local pmc word   
  .local int orig, len
  orig = pos
  len  = length tcl_code

get:
  # try to get a command name
  if pos >= len goto check
  (word, pos) = get_word(tcl_code, chars, pos)
  inc pos
  if_null word, get
  # TclWord throws an exception if you try to stringify it
  # but a TclWord isn't a comment, so we know it's not a comment
  push_eh check
    $S0 = word
  clear_eh
  if $S0 == "" goto check # this is special
  $I0 = ord $S0, 0
  if $I0 == 35 goto got_comment
check:
  .return(orig)
got_comment:
  dec pos
  .local int new_pos
  new_pos = index tcl_code, "\n", pos
  .return (new_pos)
.end

=item C<(pmc command, int pos) = get_command(string tcl_code, pmc chars, int pos)>

Tries to get a command from the Tcl code at pos, stopping at the
first character that's ord value exists in the chars hash.

    Incoming: puts [lindex "a b c" 1]
                    ^
    Outgoing: puts [lindex "a b c" 1]
                                    ^

=cut

.sub get_command
  .param string tcl_code
  .param pmc    chars
  .param int    pos

get:<commit_msg>[tcl]: Work around the no-exceptions-in-vtable-methods bug<commit_after>.HLL '_Tcl', ''
.namespace

=head1 NAME

Tcl Parser

=head1 DESCRIPTION

This is the parser that makes up the heart of Partcl. It follows
the 11 rules that are found in the Tcl manpage, available online
at <http://www.tcl.tk/man/tcl8.4/TclCmd/Tcl.htm>.

=head1 FUNCTIONS

=over 4

=item C<pmc commands = parse(string tcl_code)>

Parses the Tcl code and returns an array of TclCommand objects.
First, it performs the \<newline> substitution. Then it fetches
commands, one at a time (skipping over comments).

=cut

.sub parse
  .param string tcl_code

  .local int len
  len = length tcl_code
 
  # perform the backslash-newline substitution
  $I0 = -1
loop:
  inc $I0
  if $I0 >= len goto get_commands
  $I1 = ord tcl_code, $I0
  if $I1 != 92 goto loop # \\
  inc $I0
  $I2 = $I0
  $I1 = ord tcl_code, $I2
  if $I1 != 10 goto loop # \n
space:
  inc $I2
  if $I0 >= len goto get_commands
  $I1 = is_cclass .CCLASS_WHITESPACE, tcl_code, $I2
  if $I1 == 0 goto not_space
  goto space
not_space:
  dec $I0
  $I1 = $I2 - $I0
  substr tcl_code, $I0, $I1, " "
  dec $I1
  len -= $I1
  goto loop

get_commands:
  # an array of commands
  .local pmc commands
  $I0 = find_type "TclCommandList"
  commands = new $I0
  
  # position in the file
  .local int pos
  pos = 0
  # characters that end commands
  .local pmc chars
  chars = new .Hash
  chars[10] = 1 # \n
  chars[59] = 1 # ;
  
next_command:
  # Do we have a comment? If so, skip to the next position where
  # We might have a command.
  .local int check_pos
  check_pos = skip_comment(tcl_code, pos)
  if check_pos == pos goto done_comment

found_comment:
  pos = check_pos
  goto next_command

done_comment:
  .local pmc command
  (command, pos) = get_command(tcl_code, chars, pos)
  if_null command, done
  
  push commands, command
  goto next_command

done:
  .return(commands)
.end

=item C<int pos = skip_comment(string tcl_code, int pos)>

Checks for a comment and returns either the original pos
or the position after the comment.

    Incoming: # comment\n
              ^
    Outgoing: # comment\n
                       ^^

=cut

.sub skip_comment
  .param string tcl_code
  .param int    pos

  .local pmc chars
  chars = new .Hash
  chars[10] = 1 # \n
  
  .local pmc word   
  .local int orig, len
  orig = pos
  len  = length tcl_code

get:
  # try to get a command name
  if pos >= len goto check
  (word, pos) = get_word(tcl_code, chars, pos)
  inc pos
  if_null word, get
  # TclWord throws an exception if you try to stringify it
  # but a TclWord isn't a comment, so we know it's not a comment
  $I0 = isa word, 'TclWord'
  if $I0 goto check
  $S0 = word
  if $S0 == "" goto check # this is special
  $I0 = ord $S0, 0
  if $I0 == 35 goto got_comment
check:
  .return(orig)
got_comment:
  dec pos
  .local int new_pos
  new_pos = index tcl_code, "\n", pos
  .return (new_pos)
.end

=item C<(pmc command, int pos) = get_command(string tcl_code, pmc chars, int pos)>

Tries to get a command from the Tcl code at pos, stopping at the
first character that's ord value exists in the chars hash.

    Incoming: puts [lindex "a b c" 1]
                    ^
    Outgoing: puts [lindex "a b c" 1]
                                    ^

=cut

.sub get_command
  .param string tcl_code
  .param pmc    chars
  .param int    pos

get:
{'fraction_non_alphanumeric': 0.08868547418967587, 'fraction_numerical': 0.018607442977190875, 'mean_word_length': 2.6742006615214993, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6511972', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2437, 'n_words': 913}