<commit_before>package org.cdlib.was.ngIndexer;

import java.io.{CharArrayReader,CharArrayWriter,File,FileReader,FileWriter,Reader,StringReader,StringWriter,Writer};
import org.xml.sax.{Attributes,ContentHandler,Locator};

class NgIndexerContentHandler (useTempFile : Boolean)
  extends ContentHandler {
  
  def contentReader : Option[Reader] = 
    contentWriter match {
      case w : FileWriter      => tempFile.map(new FileReader(_));
      case w : CharArrayWriter => Some(new CharArrayReader(w.toCharArray));
      case w : StringWriter    => Some(new StringReader(w.toString));
    }

  def reader2string (r : Reader) : String = {
    var buf = new Array[Char](65536);
    var sb  = new StringBuffer();
    var i = 0;
    do {
      sb.append(buf, 0, i);
      i = r.read(buf, 0, buf.length);
    } while (i != -1);
    return sb.toString;
  }

  def contentString : Option[String] = 
    contentWriter match {
      case s : StringWriter => Some(s.toString);
      case w : Writer       => contentReader.map(reader2string(_));
    }

  val tempFile = if (useTempFile) {
    Some(File.createTempFile("ng-indexer", "data"));
  } else {
    None
  }
  
  val contentWriter : Writer = tempFile match {
    case Some(f) => new FileWriter(f);
    case None    => new StringWriter(262144);
  }

  override def characters (ch : Array[Char], start : Int, length : Int) {
    contentWriter.write(ch, start, length);
  }

  def endDocument {
    contentWriter.close;
  }
  
  def endElement (namespaceURI : String, localName : String, qName : String) = ()
  
  def endPrefixMapping (prefix : String) = ()

  def ignorableWhitespace(ch : Array[Char], start : Int, length : Int) = ()
  
  def processingInstruction(target : String, data : String) = ()
<commit_msg>strip extra whitespace<commit_after>package org.cdlib.was.ngIndexer;

import java.io.{CharArrayReader,CharArrayWriter,File,FileReader,FileWriter,Reader,StringReader,StringWriter,Writer};
import org.xml.sax.{Attributes,ContentHandler,Locator};

class NgIndexerContentHandler (useTempFile : Boolean)
  extends ContentHandler {
  
  def contentReader : Option[Reader] = 
    contentWriter match {
      case w : FileWriter      => tempFile.map(new FileReader(_));
      case w : CharArrayWriter => Some(new CharArrayReader(w.toCharArray));
      case w : StringWriter    => Some(new StringReader(w.toString));
    }

  def reader2string (r : Reader) : String = {
    var buf = new Array[Char](65536);
    var sb  = new StringBuffer();
    var i = 0;
    do {
      sb.append(buf, 0, i);
      i = r.read(buf, 0, buf.length);
    } while (i != -1);
    return sb.toString;
  }

  def contentString : Option[String] = 
    contentWriter match {
      /* replace all repeated whitespace with a single space */
      case s : StringWriter => Some(s.toString.replaceAll("""\s+""", " "));
      case w : Writer       => contentReader.map(reader2string(_).replaceAll("""\s+""", " "));
    }

  val tempFile = if (useTempFile) {
    Some(File.createTempFile("ng-indexer", "data"));
  } else {
    None
  }
  
  val contentWriter : Writer = tempFile match {
    case Some(f) => new FileWriter(f);
    case None    => new StringWriter(262144);
  }

  override def characters (ch : Array[Char], start : Int, length : Int) {
    contentWriter.write(ch, start, length);
  }

  def endDocument {
    contentWriter.close;
  }
  
  def endElement (namespaceURI : String, localName : String, qName : String) = ()
  
  def endPrefixMapping (prefix : String) = ()

  def ignorableWhitespace(ch : Array[Char], start : Int, length : Int) = ()
  
  def processingInstruction(target : String, data : String) = ()

{'fraction_non_alphanumeric': 0.13051823416506717, 'fraction_numerical': 0.009322731011790512, 'mean_word_length': 3.3171597633136094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2015856', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1138, 'n_words': 278}