<commit_before>package scalanlp.data.process;
import scalanlp.util.JavaCollections._;
import scala.collection.mutable._;

import java.io.BufferedInputStream;
import scala.io.Source;


/**
* Breaks a string into whitespace components. Uses StringTokenizer
*
* @author dlwh
*/
object WhitespaceTokenize extends (String=>Iterator[String]) {
  def apply(s:String): Iterator[String] = {
    new java.util.StringTokenizer(s).map(_.asInstanceOf[String]);
  }

  def apply(src: Source): Iterator[String] = new Iterator[String] {
    private var nextString : Option[String] = None;
    def hasNext = {
      src.hasNext && !nextString.orElse(updateString).isEmpty;
    }

    private def updateString = {
      if(!src.hasNext) {
        nextString = None;
        nextString;
      }

      val b = new StringBuilder();

      var stop = false;

      while(src.hasNext && !stop) {
        val ch = src.next;
        if(!ch.isWhitespace) {
          stop = true;
          b += ch;
        }
      }

      stop = false;
      while(src.hasNext && !stop) {
        val ch = src.next;
        if(ch.isWhitespace) {
          stop = true;
        } else { 
          b += ch;
        }
      }
      if(b.isEmpty)
        nextString = None
      else nextString = Some(b.toString);
      nextString
    }

    def next = {
      val ret = nextString.orElse(updateString).get
      nextString = None;
      ret;
    }

    
  }
}


/**
* Breaks a string into alphabetic components. Uses StringTokenizer
*
* @author dlwh
*/
object AlphaTokenize extends (String=>Iterator[String]) {
  def apply(s:String): Iterator[String] = {
    val ret = (new ArrayBuffer[String])
    ret ++= s.split("[^A-Za-z]");
    ret.elements
  }

  def apply(src: Source): Iterator[String] = new Iterator[String] {
    private var nextString : Option[String] = None;
    def hasNext = {
      src.hasNext && !nextString.orElse(updateString).isEmpty;
    }

    private def updateString = {
      if(!src.hasNext) {
        nextString = None;
        nextString;
      }

      val b = new StringBuilder();

      var stop = false;

      while(src.hasNext && !stop) {
        val ch = src.next;
        if(!ch.isWhitespace) {
          stop = true;
          b += ch;
        }
      }

      stop = false;
      while(src.hasNext && !stop) {
        val ch = src.next;
        if(ch.isWhitespace) {
          stop = true;
        } else { 
          b += ch;
        }
      }
      if(b.isEmpty)
        nextString = None
      else nextString = Some(b.toString);
      nextString
    }

    def next = {
      val ret = nextString.orElse(updateString).get
      nextString = None;
      ret;
    }

    
  }
}<commit_msg>AlphaTokenize now returns an Array.<commit_after>package scalanlp.data.process;
import scalanlp.util.JavaCollections._;
import scala.collection.mutable._;

import java.io.BufferedInputStream;
import scala.io.Source;


/**
* Breaks a string into whitespace components. Uses StringTokenizer
*
* @author dlwh
*/
object WhitespaceTokenize extends (String=>Iterator[String]) {
  def apply(s:String): Iterator[String] = {
    new java.util.StringTokenizer(s).map(_.asInstanceOf[String]);
  }

  def apply(src: Source): Iterator[String] = new Iterator[String] {
    private var nextString : Option[String] = None;
    def hasNext = {
      src.hasNext && !nextString.orElse(updateString).isEmpty;
    }

    private def updateString = {
      if(!src.hasNext) {
        nextString = None;
        nextString;
      }

      val b = new StringBuilder();

      var stop = false;

      while(src.hasNext && !stop) {
        val ch = src.next;
        if(!ch.isWhitespace) {
          stop = true;
          b += ch;
        }
      }

      stop = false;
      while(src.hasNext && !stop) {
        val ch = src.next;
        if(ch.isWhitespace) {
          stop = true;
        } else { 
          b += ch;
        }
      }
      if(b.isEmpty)
        nextString = None
      else nextString = Some(b.toString);
      nextString
    }

    def next = {
      val ret = nextString.orElse(updateString).get
      nextString = None;
      ret;
    }

    
  }
}


/**
* Breaks a string into alphabetic components. Uses StringTokenizer
*
* @author dlwh
*/
object AlphaTokenize extends (String=>Array[String]) {
  def apply(s:String) = {
    s.split("[^A-Za-z]");
  }
}
{'fraction_non_alphanumeric': 0.1238204833141542, 'fraction_numerical': 0.0, 'mean_word_length': 2.0954415954415953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7248395', 'n_tokens_mistral': 1418, 'n_tokens_neox': 1377, 'n_words': 304}