<commit_before>package com.swabunga.spell.event;

/**
 * A basic word finder, which searches text for sequences of letters.
 *
 * @author <NAME>  (<EMAIL>)
 */
public class DefaultWordFinder extends AbstractWordFinder {

  //~ Instance/static variables ...............................................

  //~ Constructors ............................................................

  /**
   * Creates a new DefaultWordFinder object.
   *
   * @param inText the String to search
   */
  public DefaultWordFinder(String inText) {
    super(inText);
  }

  //~ Methods .................................................................

  /**
   * Main method
   *
   * @param args command line input.
   */
  public static void main(String[] args) {

    String test = "A $gg ";//<B> Testing \\item<testing>and one  \\emph{two} three \\begin{four}\n \\end{dffff} five. the \\end{test}";
    //     String test = "test test\n * comm @tag comm \n test test\n*comm comm\ntest test";

    System.out.println(test);
//         DefaultWordFinder dwf = new DefaultWordFinder(test);
    //      XMLWordFinder dwf = new XMLWordFinder(test);
    TeXWordFinder dwf = new TeXWordFinder(test);
    //        JavaWordFinder dwf = new JavaWordFinder(test);

    int i = 1;
    while (dwf.hasNext()) {
      //    for (; i < 5; i++) {

      Word w = dwf.next();
      System.out.println("Word " + i++ + ": " + w.toString());
    }
//            System.out.println(dwf.toString());
//            System.out.println("C: " + dwf.current());
//            System.out.println("N: " + dwf.next());
//            System.out.println("C: " + dwf.current());
//            dwf.replace("R");
//            System.out.println(dwf.toString());
//            System.out.println("C: " + dwf.current());
//            System.out.println("N: " + dwf.next());
//            System.out.println("C: " + dwf.current());
//            System.out.println("N: " + dwf.next());
//            System.out.println("C: " + dwf.current());
//            System.out.println("N: " + dwf.next());

    //        System.out.println("Start?" + dwf.startsSentence);
    //        dwf.next();
    //        dwf.next();
    //        System.out.println(dwf.current().toString());
    //        dwf.replace("BALONEY");
    //        System.out.println(dwf.current().toString());
    //        System.out.println(dwf.toString());
    //        System.out.println(dwf.next());
  }

  /**
   * This method scans the text from the end of the last word,  and returns a
   * new Word object corresponding to the next word.
   *
   * @return the next word.
   * @throws WordNotFoundException search string contains no more words.
   */
  public Word next() {

    if (nextWord == null) {
      throw new WordNotFoundException("No more words found.");
    }

    currentWord.copy(nextWord);
    setSentenceIterator(currentWord);

    int i = currentWord.getEnd();
    boolean finished = false;
    boolean started = false;

    search:
    while (i < text.length() && !finished) {

      if (!started && isWordChar(i)) {
        nextWord.setStart(i);
        started = true;
      } else if (started && !isWordChar(i)) {
        nextWord.setText(text.substring(nextWord.getStart(), i));
        finished = true;
        break search;
      }
      i++;
    }

    if (!started) {
      nextWord = null;
    } else if (!finished) {
      nextWord.setText(text.substring(nextWord.getStart(), i));
    }

    return currentWord;
  }

}
<commit_msg>smarter next() word finding code<commit_after>package com.swabunga.spell.event;

/**
 * A basic word finder, which searches text for sequences of letters.
 * 
 * @author <NAME>  (<EMAIL>)
 */
public class DefaultWordFinder extends AbstractWordFinder {

  //~ Instance/static variables ...............................................

  //~ Constructors ............................................................

  /**
   * Creates a new DefaultWordFinder object.
   * 
   * @param inText the String to search
   */
  public DefaultWordFinder(String inText) {
    super(inText);
  }

  //~ Methods .................................................................

  /**
   * This method scans the text from the end of the last word,  and returns a
   * new Word object corresponding to the next word.
   * 
   * @return the next word.
   * @throws WordNotFoundException search string contains no more words.
   */
  public Word next() {
    if (nextWord == null) {
      throw new WordNotFoundException("No more words found.");
    }
    currentWord.copy(nextWord);
    setSentenceIterator(currentWord);

    int i = currentWord.getEnd();
    boolean finished = false;

    while (i < text.length() && !finished) {
      if (isWordChar(i)) {
        nextWord.setStart(i);
        int end = getNextWordEnd(text, i);
        nextWord.setText(text.substring(i, end));
        finished = true;
      }
      i++;
    }
    if (!finished)
      nextWord = null;

    return currentWord;
  }

  /**
   * Returns the position in the string <em>after</em> the end of the next word.
   * Note that this return value should not be used as an index into the string
   * without checking first that it is in range, since it is possible for the
   * value <code>text.length()</code> to be returned by this method.
   */
  private int getNextWordEnd(String text, int startPos) {
    // If we're dealing with a possible 'internet word' we need to provide
    // some special handling
    if (SpellChecker.isINETWord(text.substring(startPos))) {
      for (int i = startPos; i < text.length(); i++) {
        char ch = text.charAt(i);
        if (Character.isLetterOrDigit(ch))
          continue;

        if (ch == '\r' || ch == '\n')
          return i;
        // Chop off any characters that might be enclosing the 'internet word'. eg ',",),]
        if (Character.isSpaceChar(ch))
          if (i > 0 && Character.isLetterOrDigit(text.charAt(i - 1)))
            return i;
          else
            return i - 1;
      }
      return text.length();
    } else {
      for (int i = startPos; i < text.length(); i++) {
        if (!isWordChar(i))
          return i;
      }
      return text.length();
    }
  }
}

{'fraction_non_alphanumeric': 0.19036845507433742, 'fraction_numerical': 0.0008080155138978668, 'mean_word_length': 1.8652777777777778, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1686674', 'n_tokens_mistral': 1826, 'n_tokens_neox': 1790, 'n_words': 487}