<commit_before>//To patch the generated ReqTTokenMaker.java, do the following

//search replace:
zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
//with:
zzCurrentPos = zzMarkedPos = s.offset;

//search delete these two methods:

/**
   * Refills the input buffer.
   *
   * @return      <code>false</code>, iff there was new input.
   * 
   * @exception   java.io.IOException  if any I/O-Error occurs
   */
  private boolean zzRefill() throws java.io.IOException {
  
// delete until but not including next /**
  

  /**
   * Resets the scanner to read from a new input stream.
   * Does not close the old reader.
   *
   * All internal variables are reset, the old input stream 
   * <b>cannot</b> be reused (internal buffer is discarded and lost).
   * Lexical state is set to <tt>ZZ_INITIAL</tt>.
   *
   * Internal scan buffer is resized down to its initial length, if it has grown.
   *
   * @param reader   the new input stream 
   */
  public final void yyreset(java.io.Reader reader) {
  
// delete until but not including next /**<commit_msg>remove comments<commit_after>//To patch the generated ReqTTokenMaker.java, do the following

//search for:

zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;

//and replace with:

zzCurrentPos = zzMarkedPos = s.offset;

//search for and delete these two methods:

private boolean zzRefill() throws java.io.IOException {
  
public final void yyreset(java.io.Reader reader) {
  
// delete from the /** opening comment until end of method just before next /**
{'fraction_non_alphanumeric': 0.0967741935483871, 'fraction_numerical': 0.0, 'mean_word_length': 3.1758241758241756, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3612477', 'n_tokens_mistral': 468, 'n_tokens_neox': 460, 'n_words': 172}