<commit_before>package org.mitre.jcarafe.util

import org.mitre.jcarafe.lexer._
import org.mitre.jcarafe.tokenizer._
import GenTokerConstants._
import java.io._

object StripTags {

  def main(args: Array[String]) = {
    if (args.length == 2) {
      val sr = new java.io.FileInputStream(args(0))
      val os = new java.io.OutputStreamWriter(new java.io.BufferedOutputStream(new java.io.FileOutputStream(new java.io.File(args(1)))), "UTF-8")
      val parser = new GenToker(sr)
      var c = true
      while (c) {
	val t : Token = parser.getNextToken()
	t.kind match {
          case EOF => c = false
          case TAGSTART => 
          case TAGEND => 
          case _ => os.write(t.image)
	}
      }
    } else println("usage: java -cp jcarafe...jar org.mitre.jcarafe.util.StripTags <input file> <output file>")
  }


}<commit_msg>Update to fix lexical scoping conflict<commit_after>package org.mitre.jcarafe.util

import org.mitre.jcarafe.lexer._
import GenTokerConstants._
import java.io._

object StripTags {

  def main(args: Array[String]) = {
    if (args.length == 2) {
      val sr = new java.io.FileInputStream(args(0))
      val os = new java.io.OutputStreamWriter(new java.io.BufferedOutputStream(new java.io.FileOutputStream(new java.io.File(args(1)))), "UTF-8")
      val parser = new GenToker(sr)
      var c = true
      while (c) {
	val t : org.mitre.jcarafe.lexer.Token = parser.getNextToken()
	t.kind match {
          case EOF => c = false
          case TAGSTART => 
          case TAGEND => 
          case _ => os.write(t.image)
	}
      }
    } else println("usage: java -cp jcarafe...jar org.mitre.jcarafe.util.StripTags <input file> <output file>")
  }


}
{'fraction_non_alphanumeric': 0.13033175355450238, 'fraction_numerical': 0.004739336492890996, 'mean_word_length': 2.7367256637168142, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3402794', 'n_tokens_mistral': 603, 'n_tokens_neox': 578, 'n_words': 133}