<commit_before>package com.nrinaudo.csv

import java.io.IOException

import scala.collection.AbstractIterator
import scala.collection.mutable.ArrayBuffer
import scala.io.Source

/** Defines the various possible states of a CSV parser. */
private object CsvIterator {
  object Status {
    case object Normal extends Status
    case object Escaping extends Status
    case object LeavingEscape extends Status
  }
  sealed trait Status
}

private[csv] class CsvIterator(data: Source, separator: Char)
  extends AbstractIterator[ArrayBuffer[String]] with Iterator[ArrayBuffer[String]] with AutoCloseable {
  import CsvIterator._

  /** Used to aggregate the content of the current cell. */
  private val cell = new StringBuilder
  /** Used to aggregate the content of the current row. */
  private val row = ArrayBuffer[String]()
  /** Parser status. */
  private var status: Status = Status.Normal
  /** Iterator on the CSV data. We need this to be buffered to deal with possible `\r\n` sequences. */
  private val input: BufferedIterator[Char] = data.buffered
  /** Number of whitespace found at the end of and escaped cell. */
  private var wCount = 0

  /** Appends the content of current cell to the current row. */
  private def appendCell() = {
    row += cell.result()
    cell.clear()
    row
  }

  /** Checks whether the specified character is a line break.
    *<commit_msg>Fix compilation issues with Java 6: AutoCloseable wasn't actually available before Java 7.<commit_after>package com.nrinaudo.csv

import java.io.{Closeable, IOException}

import scala.collection.AbstractIterator
import scala.collection.mutable.ArrayBuffer
import scala.io.Source

/** Defines the various possible states of a CSV parser. */
private object CsvIterator {
  object Status {
    case object Normal extends Status
    case object Escaping extends Status
    case object LeavingEscape extends Status
  }
  sealed trait Status
}

private[csv] class CsvIterator(data: Source, separator: Char)
  extends AbstractIterator[ArrayBuffer[String]] with Iterator[ArrayBuffer[String]] with Closeable {
  import CsvIterator._

  /** Used to aggregate the content of the current cell. */
  private val cell = new StringBuilder
  /** Used to aggregate the content of the current row. */
  private val row = ArrayBuffer[String]()
  /** Parser status. */
  private var status: Status = Status.Normal
  /** Iterator on the CSV data. We need this to be buffered to deal with possible `\r\n` sequences. */
  private val input: BufferedIterator[Char] = data.buffered
  /** Number of whitespace found at the end of and escaped cell. */
  private var wCount = 0

  /** Appends the content of current cell to the current row. */
  private def appendCell() = {
    row += cell.result()
    cell.clear()
    row
  }

  /** Checks whether the specified character is a line break.
    *
{'fraction_non_alphanumeric': 0.08274894810659186, 'fraction_numerical': 0.001402524544179523, 'mean_word_length': 3.684729064039409, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '368737', 'n_tokens_mistral': 772, 'n_tokens_neox': 756, 'n_words': 319}