<commit_before>/*
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package edu.brandeis.cs.steele.wn;
import java.util.Iterator;
import java.util.NoSuchElementException;

/**
 * XXX borrowed from Apache Harmony java.util.StringTokenizer<br><br>
 *
 * CharSequenceTokenizer is used to break a string apart into tokens.  Lighter
 * than a Scanner, more features than java.util.StringTokenizer, with full
 * support for CharSequences.
 */
public class CharSequenceTokenizer implements Iterator<CharSequence> {
  private final CharSequence string;
  private String delimiters;
  private int position;

  /**
   * Constructs a new CharSequenceTokenizer for string using whitespace as
   * the delimiter.
   * 
   * @param string the CharSequence to be tokenized
   */
  public CharSequenceTokenizer(final CharSequence string) {
    this(string, " \t\n\r\f");
  }

  /**
   * Constructs a new CharSequenceTokenizer for string using the specified
   * delimiters and returning delimiters as tokens when specified.
   * 
   * @param string the string to be tokenized
   * @param delimiters the delimiters to use
   */
  public CharSequenceTokenizer(final CharSequence string, final String delimiters) {
    if (string != null) {
      this.string = string;
      this.delimiters = delimiters;
      this.position = 0;
    } else {
      throw new NullPointerException();
    }
  }

  /**
   * Returns the number of unprocessed tokens remaining in the string.
   *
   * @return number of tokens that can be retreived before an exception will
   *         result
   */
  public int countTokens() {
    int count = 0;
    boolean inToken = false;
    for (int i = position, length = string.length(); i < length; i++) {
      if (delimiters.indexOf(string.charAt(i), 0) >= 0) {
        if (inToken) {
          count++;
          inToken = false;<commit_msg>javadoc comments<commit_after>/*
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package edu.brandeis.cs.steele.wn;
import java.util.Iterator;
import java.util.NoSuchElementException;

/**
 * XXX borrowed from Apache Harmony java.util.StringTokenizer<br><br>
 *
 * <code>CharSequenceTokenizer</code> is used to break a string apart into tokens.  Lighter
 * than a {@link java.util.Scanner}, more features than {@link java.util.StringTokenizer}, with full
 * support for {@link CharSequence}s.
 */
public class CharSequenceTokenizer implements Iterator<CharSequence> {
  private final CharSequence string;
  private String delimiters;
  private int position;

  /**
   * Constructs a new <code>CharSequenceTokenizer</code> for <code>string</code> using whitespace as
   * the delimiter.
   * 
   * @param string the CharSequence to be tokenized
   */
  public CharSequenceTokenizer(final CharSequence string) {
    this(string, " \t\n\r\f");
  }

  /**
   * Constructs a new <code>CharSequenceTokenizer</code> for <code>string</code> using the specified
   * <code>delimiters</code> and returning delimiters as tokens when specified.
   * 
   * @param string the string to be tokenized
   * @param delimiters the delimiters to use
   */
  public CharSequenceTokenizer(final CharSequence string, final String delimiters) {
    if (string != null) {
      this.string = string;
      this.delimiters = delimiters;
      this.position = 0;
    } else {
      throw new NullPointerException();
    }
  }

  /**
   * Returns the number of unprocessed tokens remaining in the string.
   *
   * @return number of tokens that can be retreived before an exception will
   *         result
   */
  public int countTokens() {
    int count = 0;
    boolean inToken = false;
    for (int i = position, length = string.length(); i < length; i++) {
      if (delimiters.indexOf(string.charAt(i), 0) >= 0) {
        if (inToken) {
          count++;
          inToken = false;
{'fraction_non_alphanumeric': 0.08384918401800788, 'fraction_numerical': 0.0030013130744700805, 'mean_word_length': 3.3034705407586764, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1214311', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1464, 'n_words': 577}