<commit_before>package de.knutwalker.ntparser

import java.io.InputStream
import java.lang.{ StringBuilder, Iterable ⇒ JIterable }
import java.nio.charset.{ StandardCharsets, Charset }
import java.util.{ Iterator ⇒ JIterator }

import org.slf4j.LoggerFactory

import scala.annotation.{ switch, tailrec }
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.collection.GenIterable
import scala.util.Try

final class NtParser {

  private[this] val logger = LoggerFactory.getLogger(classOf[NtParser])

  private[this] var input: Array[Char] = new Array[Char](1024)

  private[this] var pos = 0
  private[this] var max = 0
  private[this] var cursor: Char = 0
  private[this] var lineNo = -1

  private[this] val sb = new StringBuilder
  private[this] val nodes: ListBuffer[Node] = ListBuffer.empty
  private[this] val statement: Array[Node] = new Array[Node](3)

  def parseOpt(line: String): Option[Statement] = Option(parseOrNull(line))

  def parseOpt(line: String, lineNumber: Int): Option[Statement] = Option(parseOrNull(line, lineNumber))

  def parseTry(line: String): Try[Option[Statement]] = Try(Option(parse(line)))

  def parseTry(line: String, lineNumber: Int): Try[Option[Statement]] = Try(Option(parse(line, lineNumber)))

  def parseOrNull(line: String): Statement = {
    try parse(line) catch {
      case pe: ParseError ⇒
        logger.warn(pe.getMessage, pe)
        null
    }
  }

  def parseOrNull(line: String, lineNumber: Int): Statement = {
    try parse(line, lineNo) catch {
      case pe: ParseError ⇒
        logger.warn(pe.getMessage, pe)
        null
    }
  }

  @throws[ParseError]("ParseError if a line could not be parsed")
  def parse(line: String): Statement = {

    lineNo = -1
    if (line.isEmpty) null
    else {
      reset(line)
      Line()
    }
  }

  @throws[ParseError]("ParseError if a line could not be parsed")
  def parse(line: String, lineNumber: Int): Statement = {
    lineNo = lineNumber
    if (line.isEmpty) null
    else {
      reset(line)
      Line()
    }
  }

  private[this] def Line(): Statement = {
    ws()
    (cursor: @switch) match {
      case '<' ⇒ TripleLine()
      case '_' ⇒ TripleLine()
      case '#' ⇒ // comment line
      case END ⇒ // empty line
      case _   ⇒ error(LINE_BEGIN)
    }

    if ((statement(0) ne null) && (statement(1) ne null) && (statement(2) ne null)) {<commit_msg>Inline Char.END to allow emitting of switch statement<commit_after>package de.knutwalker.ntparser

import java.io.InputStream
import java.lang.{ StringBuilder, Iterable ⇒ JIterable }
import java.nio.charset.{ StandardCharsets, Charset }
import java.util.{ Iterator ⇒ JIterator }

import org.slf4j.LoggerFactory

import scala.annotation.{ switch, tailrec }
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.collection.GenIterable
import scala.util.Try

final class NtParser {

  private[this] val logger = LoggerFactory.getLogger(classOf[NtParser])

  private[this] var input: Array[Char] = new Array[Char](1024)

  private[this] var pos = 0
  private[this] var max = 0
  private[this] var cursor: Char = 0
  private[this] var lineNo = -1

  private[this] val sb = new StringBuilder
  private[this] val nodes: ListBuffer[Node] = ListBuffer.empty
  private[this] val statement: Array[Node] = new Array[Node](3)

  def parseOpt(line: String): Option[Statement] = Option(parseOrNull(line))

  def parseOpt(line: String, lineNumber: Int): Option[Statement] = Option(parseOrNull(line, lineNumber))

  def parseTry(line: String): Try[Option[Statement]] = Try(Option(parse(line)))

  def parseTry(line: String, lineNumber: Int): Try[Option[Statement]] = Try(Option(parse(line, lineNumber)))

  def parseOrNull(line: String): Statement = {
    try parse(line) catch {
      case pe: ParseError ⇒
        logger.warn(pe.getMessage, pe)
        null
    }
  }

  def parseOrNull(line: String, lineNumber: Int): Statement = {
    try parse(line, lineNo) catch {
      case pe: ParseError ⇒
        logger.warn(pe.getMessage, pe)
        null
    }
  }

  @throws[ParseError]("ParseError if a line could not be parsed")
  def parse(line: String): Statement = {
    lineNo = -1
    if (line.isEmpty) null
    else {
      reset(line)
      Line()
    }
  }

  @throws[ParseError]("ParseError if a line could not be parsed")
  def parse(line: String, lineNumber: Int): Statement = {
    lineNo = lineNumber
    if (line.isEmpty) null
    else {
      reset(line)
      Line()
    }
  }

  private[this] def Line(): Statement = {
    ws()
    (cursor: @switch) match {
      case '<' ⇒ TripleLine()
      case '_' ⇒ TripleLine()
      case '#' ⇒ // comment line
      case '\u0000' ⇒ // empty line, inline char to allow switch statement
      case _   ⇒ error(LINE_BEGIN)
    }

    if ((statement(0) ne null) && (statement(1) ne null) && (statement(2) ne null)) {
{'fraction_non_alphanumeric': 0.1228856735276136, 'fraction_numerical': 0.0065212961076013855, 'mean_word_length': 3.515179392824287, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7054424', 'n_tokens_mistral': 1597, 'n_tokens_neox': 1568, 'n_words': 441}