<commit_before>.HLL 'Tcl', 'tcl_group'
.namespace

.sub '&split'
  .param pmc argv :slurpy

  .local int argc
  argc = argv

  if argc == 0 goto bad_args
  if argc >  2 goto bad_args

  .local string splitchars,splitstring
  splitstring = argv[0]
  if splitstring != "" goto check_splitchars
  .return ('')

check_splitchars:
  # RT#40755: Make this the default whitespace splitchars.
  splitchars = " \r\n\t"

  if argc == 1 goto got_splitchars
  splitchars = argv[1]


got_splitchars:
  if splitchars == '' goto split_empty

  .local pmc charHash 
  charHash = new .Hash

  .local int sc_len, pos
  .local string split_char
  sc_len = length splitchars
  pos = 0
hash_loop:
  if pos == sc_len goto done_hash_loop
  split_char = substr splitchars, pos, 1

  charHash[split_char] = 1
  inc pos
  goto hash_loop

done_hash_loop:

  # Loop over each character in the string. Is it one of the split
  # chars?
  .local pmc results
  results = new .TclList
  .local int str_len
  str_len = length splitstring

  .local int begin,len,last_match,result_key
  .local string check_char
  last_match = -1
  pos = 0
  result_key = 0

split_loop:
  if str_len == pos goto split_done
  split_char = substr splitstring, pos, 1 
  $I1 = exists charHash[split_char]
  unless $I1 goto next_split_loop
  begin = last_match + 1
  len = pos - begin
  check_char = substr splitstring, begin, len
  results[result_key] = check_char
  last_match = begin + len
  inc result_key
next_split_loop:
  inc pos
  goto split_loop
split_done:
  if last_match == str_len goto split_really_done
  begin = last_match + 1
  len = str_len - begin
  check_char = substr splitstring, begin, len
  results[result_key] = check_char
  inc result_key

split_really_done:
  # results = result_key # RT#40756: another TclList hack, truncate the list to the right
  # number of elements...
  .return (results)

split_empty:
  results = new .TclList
  str_len = length splitstring
  pos = 0
split_empty_loop:
  if pos == str_len goto empty_done
  split_char = substr splitstring, pos, 1
  results[pos] = split_char
  inc pos
  goto split_empty_loop

empty_done:
  .return(results)

bad_args:
  tcl_error 'wrong # args: should be "split string ?splitChars?"'<commit_msg>RT # 40756 - whoops. remove line, don't comment it out.<commit_after>.HLL 'Tcl', 'tcl_group'
.namespace

.sub '&split'
  .param pmc argv :slurpy

  .local int argc
  argc = argv

  if argc == 0 goto bad_args
  if argc >  2 goto bad_args

  .local string splitchars,splitstring
  splitstring = argv[0]
  if splitstring != "" goto check_splitchars
  .return ('')

check_splitchars:
  # RT#40755: Make this the default whitespace splitchars.
  splitchars = " \r\n\t"

  if argc == 1 goto got_splitchars
  splitchars = argv[1]


got_splitchars:
  if splitchars == '' goto split_empty

  .local pmc charHash 
  charHash = new .Hash

  .local int sc_len, pos
  .local string split_char
  sc_len = length splitchars
  pos = 0
hash_loop:
  if pos == sc_len goto done_hash_loop
  split_char = substr splitchars, pos, 1

  charHash[split_char] = 1
  inc pos
  goto hash_loop

done_hash_loop:

  # Loop over each character in the string. Is it one of the split
  # chars?
  .local pmc results
  results = new .TclList
  .local int str_len
  str_len = length splitstring

  .local int begin,len,last_match,result_key
  .local string check_char
  last_match = -1
  pos = 0
  result_key = 0

split_loop:
  if str_len == pos goto split_done
  split_char = substr splitstring, pos, 1 
  $I1 = exists charHash[split_char]
  unless $I1 goto next_split_loop
  begin = last_match + 1
  len = pos - begin
  check_char = substr splitstring, begin, len
  results[result_key] = check_char
  last_match = begin + len
  inc result_key
next_split_loop:
  inc pos
  goto split_loop
split_done:
  if last_match == str_len goto split_really_done
  begin = last_match + 1
  len = str_len - begin
  check_char = substr splitstring, begin, len
  results[result_key] = check_char
  inc result_key

split_really_done:
  .return (results)

split_empty:
  results = new .TclList
  str_len = length splitstring
  pos = 0
split_empty_loop:
  if pos == str_len goto empty_done
  split_char = substr splitstring, pos, 1
  results[pos] = split_char
  inc pos
  goto split_empty_loop

empty_done:
  .return(results)

bad_args:
  tcl_error 'wrong # args: should be "split string ?splitChars?"'
{'fraction_non_alphanumeric': 0.0768526989935956, 'fraction_numerical': 0.012808783165599268, 'mean_word_length': 3.5269151138716355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1773582', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1537, 'n_words': 488}