<commit_before>﻿// --------------------------------------------------------------------------------------
// CSV type provider - runtime components (parsing and type representing CSV)
// --------------------------------------------------------------------------------------

namespace FSharp.Data.Runtime

open System
open System.ComponentModel
open System.Collections.Generic
open System.IO
open System.Runtime.InteropServices
open System.Text
open FSharp.Data.Runtime
open FSharp.Data.Runtime.IO

// --------------------------------------------------------------------------------------

// Parser for the CSV format 
module internal CsvReader = 

  /// Lazily reads the specified CSV file using the specified separators
  /// (Handles most of the RFC 4180 - most notably quoted values and also
  /// quoted newline characters in columns)
  let readCsvFile (reader:TextReader) (separators:string) quote =

    let inline (|Char|) (n:int) = char n
    let inline (|Quote|_|) (n:int) = if char n = quote then Some() else None
    
    let separators = separators.ToCharArray()
    let inline (|Separator|_|) (n:int) =
      if separators.Length = 1 then 
        if (char n) = separators.[0] then Some() else None
      else
        if Array.exists ((=) (char n)) separators then Some() else None

    /// Read quoted string value until the end (ends with end of stream or
    /// the " character, which can be encoded using double ")
    let rec readString (chars:StringBuilder) = 
      match reader.Read() with
      | -1 -> chars
      | Quote when reader.Peek() = int quote ->
          reader.Read() |> ignore
          readString (chars.Append quote)
      | Quote -> chars
      | Char c -> readString (chars.Append c)

    /// Reads a line with data that are separated using specified separators
    /// and may be quoted. Ends with newline or end of input.
    let rec readLine data (chars:StringBuilder) = 
      match reader.Read() with
      | -1 | Char '\r' | Char '\n' -> 
          let item = chars.ToString()
          item::data
      | Separator -> 
          let item = chars.ToString()
          readLine (item::data) (StringBuilder())
      | Quote ->
          readLine data (readString chars)
      | Char c ->
          readLine data (chars.Append c)

    /// Reads multiple lines from the input, skipping newline characters
    let rec readLines lineNumber = seq {
      match reader.Peek() with
      | -1 -> ()
      | Char '\r' | Char '\n' -> 
          reader.Read() |> ignore
          yield! readLines lineNumber
      | _ -> 
          yield readLine [] (StringBuilder()) |> List.rev |> Array.ofList, lineNumber
          yield! readLines (lineNumber + 1) }

    readLines 0

// --------------------------------------------------------------------------------------

module private CsvHelpers = 

  let inline tryConvert (stringArrayToRow:Func<obj, string[], 'RowType>) this row = 
    try 
      stringArrayToRow.Invoke(this, row) |> Choice1Of2 
    with exn -> 
      Choice2Of2 exn

// --------------------------------------------------------------------------------------

/// An enumerable that will return elements from the 'firstSeq' first time it
/// is accessed and then will call 'nextSeq' each time for all future GetEnumerator calls
type private ReentrantEnumerable<'T>(firstSeq:seq<'T>, nextSeq:unit -> seq<'T>) =
  let mutable first = true
  interface seq<'T> with
    member x.GetEnumerator() = 
      if first then 
        first <- false
        firstSeq.GetEnumerator()
      else nextSeq().GetEnumerator()
  interface System.Collections.IEnumerable with
    member x.GetEnumerator() = (x :> seq<'T>).GetEnumerator() :> System.Collections.IEnumerator

// --------------------------------------------------------------------------------------

/// [omit]
type private parsedFirstLine = {
    Reader: TextReader
    FirstLine: string[] * int
    Headers: string[] option
    LineIterator: IEnumerator<string[] * int> 
    ColumnCount: int
  }

/// [omit]
type CsvFile<'RowType> private (rowToStringArray:Func<'RowType,string[]>, disposer:IDisposable, rows:seq<'RowType>, headers, numberOfColumns, separators, quote) =

  /// The rows with data
  member __.Rows = rows
  /// The names of the columns
  member __.Headers = headers
  /// The number of columns
  member __.NumberOfColumns = numberOfColumns
  /// The character(s) used as column separator(s)
  member __.Separators = separators
  /// The quotation mark use for surrounding values containing separator chars
  member __.Quote = quote
  
  interface IDisposable with
    member __.Dispose() = 
      if disposer <> null then disposer.Dispose()

  /// [omit]
  [<EditorBrowsableAttribute(EditorBrowsableState.Never)>]
  [<CompilerMessageAttribute("This method is intended for use in generated code only.", 10001, IsHidden=true, IsError=false)>]
  static member CreateNonReentrant (stringArrayToRow, rowToStringArray, reader:TextReader, separators, quote, hasHeaders, ignoreErrors, cacheRows) =    
    let uncachedCsv = new CsvFile<'RowType>(stringArrayToRow, rowToStringArray, Func<_>(fun _ -> reader), separators, quote, hasHeaders, ignoreErrors)
    if cacheRows then uncachedCsv.Cache() else uncachedCsv

  /// [omit]
  new (stringArrayToRow, rowToStringArray, readerFunc:Func<TextReader>, separators, quote, hasHeaders, ignoreErrors) =

    let defaultSeparator = String.IsNullOrEmpty separators
    let mutable separators = if defaultSeparator then "," else separators

    // Track created Readers so that we can dispose of all of them
    let disposeFuncs = new ResizeArray<_>()
    let disposer = 
      { new IDisposable with
          member x.Dispose() = Seq.iter (fun f -> f()) disposeFuncs }

    let detectFirstLine separators =
      // Call 'readerFunc' to get the first iterator and read the first line
      let firstReader = readerFunc.Invoke()
      let linesIterator = (CsvReader.readCsvFile firstReader separators quote).GetEnumerator()  
      let firstLine = 
        if linesIterator.MoveNext() then
          linesIterator.Current
        else
          // If it does not have any lines, that's wrong...
          linesIterator.Dispose()
          if hasHeaders then failwithf "Invalid CSV file: header row not found" 
          else failwithf "Invalid CSV file: no data rows found"

      // Get headers and count columns (if no headers, use the first line)
      let headers = 
        if not hasHeaders then None
        else firstLine |> fst |> Array.map (fun columnName -> columnName.Trim()) |> Some
      let numberOfColumns =
        let (Some headers, _ | _, (headers, _)) = headers, firstLine
        headers |> Array.length
        
      {Reader = firstReader; FirstLine = firstLine; Headers = headers; LineIterator = linesIterator; ColumnCount = numberOfColumns}

    let firstResult = detectFirstLine(separators)
        
    disposeFuncs.Add(firstResult.Reader.Dispose)

    let mutable firstLine = firstResult.FirstLine
    let mutable numberOfColumns = firstResult.ColumnCount
    let mutable headers = firstResult.Headers
    let mutable linesIterator = firstResult.LineIterator

    // Auto-Detect tab separated files that may not have .TSV extension when no explicit separators defined
    let probablyTabSeparated = numberOfColumns < 2 && defaultSeparator = true && fst firstLine |> Seq.exists (fun c -> c.IndexOf("\t") > -1)
    if probablyTabSeparated = true then
      // Re-parse the first line using tab separator
      separators <- "\t"
      let firstResult = detectFirstLine(separators)
      firstLine <- firstResult.FirstLine
      numberOfColumns <- firstResult.ColumnCount
      headers <- firstResult.Headers
      linesIterator <- firstResult.LineIterator

    new CsvFile<'RowType>(linesIterator, hasHeaders, firstLine, readerFunc, ignoreErrors, stringArrayToRow, rowToStringArray, disposer, headers, numberOfColumns, separators, quote)    

  /// [omit]
  new ((linesIterator:IEnumerator<string[] * int>), hasHeaders, firstLine, (readerFunc:Func<TextReader>), ignoreErrors, stringArrayToRow, rowToStringArray, disposer, headers, numberOfColumns, separators, quote) as this =
    // Create sequence that is exposed as 'Data' - on the first read, finish
    // reading the opened reader; on future reads, get a new reader (and skip headers)
    let firstData = seq {
      use ls = linesIterator
      if not hasHeaders then yield firstLine
      while ls.MoveNext() do yield ls.Current }
    let nextData () = seq { 
      let reader = readerFunc.Invoke()
      let csv = CsvReader.readCsvFile reader separators quote
      yield! if hasHeaders then Seq.skip 1 csv else csv }
    let data = ReentrantEnumerable<_>(firstData, nextData)

    // Return data with parsed columns
    let data = seq {
      for row, lineNumber in data do
        if row.Length <> numberOfColumns then
          // Ignore rows with different number of columns when ignoreErrors is set to true
          if not ignoreErrors then
            let lineNumber = if hasHeaders then lineNumber else lineNumber + 1
            failwithf "Couldn't parse row %d according to schema: Expected %d columns, got %d" lineNumber numberOfColumns row.Length
        else
          // Always ignore empty rows<commit_msg>Refactor TSV auto-detect to be less clumsy<commit_after>﻿// --------------------------------------------------------------------------------------
// CSV type provider - runtime components (parsing and type representing CSV)
// --------------------------------------------------------------------------------------

namespace FSharp.Data.Runtime

open System
open System.ComponentModel
open System.Collections.Generic
open System.IO
open System.Runtime.InteropServices
open System.Text
open FSharp.Data.Runtime
open FSharp.Data.Runtime.IO

// --------------------------------------------------------------------------------------

// Parser for the CSV format 
module internal CsvReader = 

  /// Lazily reads the specified CSV file using the specified separators
  /// (Handles most of the RFC 4180 - most notably quoted values and also
  /// quoted newline characters in columns)
  let readCsvFile (reader:TextReader) (separators:string) quote =

    let inline (|Char|) (n:int) = char n
    let inline (|Quote|_|) (n:int) = if char n = quote then Some() else None
    
    let separators = separators.ToCharArray()
    let inline (|Separator|_|) (n:int) =
      if separators.Length = 1 then 
        if (char n) = separators.[0] then Some() else None
      else
        if Array.exists ((=) (char n)) separators then Some() else None

    /// Read quoted string value until the end (ends with end of stream or
    /// the " character, which can be encoded using double ")
    let rec readString (chars:StringBuilder) = 
      match reader.Read() with
      | -1 -> chars
      | Quote when reader.Peek() = int quote ->
          reader.Read() |> ignore
          readString (chars.Append quote)
      | Quote -> chars
      | Char c -> readString (chars.Append c)

    /// Reads a line with data that are separated using specified separators
    /// and may be quoted. Ends with newline or end of input.
    let rec readLine data (chars:StringBuilder) = 
      match reader.Read() with
      | -1 | Char '\r' | Char '\n' -> 
          let item = chars.ToString()
          item::data
      | Separator -> 
          let item = chars.ToString()
          readLine (item::data) (StringBuilder())
      | Quote ->
          readLine data (readString chars)
      | Char c ->
          readLine data (chars.Append c)

    /// Reads multiple lines from the input, skipping newline characters
    let rec readLines lineNumber = seq {
      match reader.Peek() with
      | -1 -> ()
      | Char '\r' | Char '\n' -> 
          reader.Read() |> ignore
          yield! readLines lineNumber
      | _ -> 
          yield readLine [] (StringBuilder()) |> List.rev |> Array.ofList, lineNumber
          yield! readLines (lineNumber + 1) }

    readLines 0

// --------------------------------------------------------------------------------------

module private CsvHelpers = 

  let inline tryConvert (stringArrayToRow:Func<obj, string[], 'RowType>) this row = 
    try 
      stringArrayToRow.Invoke(this, row) |> Choice1Of2 
    with exn -> 
      Choice2Of2 exn

// --------------------------------------------------------------------------------------

/// An enumerable that will return elements from the 'firstSeq' first time it
/// is accessed and then will call 'nextSeq' each time for all future GetEnumerator calls
type private ReentrantEnumerable<'T>(firstSeq:seq<'T>, nextSeq:unit -> seq<'T>) =
  let mutable first = true
  interface seq<'T> with
    member x.GetEnumerator() = 
      if first then 
        first <- false
        firstSeq.GetEnumerator()
      else nextSeq().GetEnumerator()
  interface System.Collections.IEnumerable with
    member x.GetEnumerator() = (x :> seq<'T>).GetEnumerator() :> System.Collections.IEnumerator

// --------------------------------------------------------------------------------------

/// [omit]
type private parsedFirstLine = {
    Reader: TextReader
    FirstLine: string[] * int
    Headers: string[] option
    LineIterator: IEnumerator<string[] * int> 
    ColumnCount: int
  }

/// [omit]
type CsvFile<'RowType> private (rowToStringArray:Func<'RowType,string[]>, disposer:IDisposable, rows:seq<'RowType>, headers, numberOfColumns, separators, quote) =

  /// The rows with data
  member __.Rows = rows
  /// The names of the columns
  member __.Headers = headers
  /// The number of columns
  member __.NumberOfColumns = numberOfColumns
  /// The character(s) used as column separator(s)
  member __.Separators = separators
  /// The quotation mark use for surrounding values containing separator chars
  member __.Quote = quote
  
  interface IDisposable with
    member __.Dispose() = 
      if disposer <> null then disposer.Dispose()

  /// [omit]
  [<EditorBrowsableAttribute(EditorBrowsableState.Never)>]
  [<CompilerMessageAttribute("This method is intended for use in generated code only.", 10001, IsHidden=true, IsError=false)>]
  static member CreateNonReentrant (stringArrayToRow, rowToStringArray, reader:TextReader, separators, quote, hasHeaders, ignoreErrors, cacheRows) =    
    let uncachedCsv = new CsvFile<'RowType>(stringArrayToRow, rowToStringArray, Func<_>(fun _ -> reader), separators, quote, hasHeaders, ignoreErrors)
    if cacheRows then uncachedCsv.Cache() else uncachedCsv

  /// [omit]
  new (stringArrayToRow, rowToStringArray, readerFunc:Func<TextReader>, separators, quote, hasHeaders, ignoreErrors) =

    let defaultSeparator = String.IsNullOrEmpty separators
    let mutable separators = if defaultSeparator then "," else separators

    // Track created Readers so that we can dispose of all of them
    let disposeFuncs = new ResizeArray<_>()
    let disposer = 
      { new IDisposable with
          member x.Dispose() = Seq.iter (fun f -> f()) disposeFuncs }

    let detectFirstLine separators =
      // Call 'readerFunc' to get the first iterator and read the first line
      let firstReader = readerFunc.Invoke()
      let linesIterator = (CsvReader.readCsvFile firstReader separators quote).GetEnumerator()  
      let firstLine = 
        if linesIterator.MoveNext() then
          linesIterator.Current
        else
          // If it does not have any lines, that's wrong...
          linesIterator.Dispose()
          if hasHeaders then failwithf "Invalid CSV file: header row not found" 
          else failwithf "Invalid CSV file: no data rows found"

      // Get headers and count columns (if no headers, use the first line)
      let headers = 
        if not hasHeaders then None
        else firstLine |> fst |> Array.map (fun columnName -> columnName.Trim()) |> Some
      let numberOfColumns =
        let (Some headers, _ | _, (headers, _)) = headers, firstLine
        headers |> Array.length
        
      {Reader = firstReader; FirstLine = firstLine; Headers = headers; LineIterator = linesIterator; ColumnCount = numberOfColumns}

    let firstResult = detectFirstLine(separators)
        
    disposeFuncs.Add(firstResult.Reader.Dispose)

    // Auto-Detect tab separated files that may not have .TSV extension when no explicit separators defined
    let probablyTabSeparated =
      firstResult.ColumnCount < 2 && defaultSeparator = true &&
      fst firstResult.FirstLine |> Seq.exists (fun c -> c.IndexOf("\t") > -1)

    let finalResult =
      if probablyTabSeparated then
        separators <- "\t"
        detectFirstLine(separators) // Re-parse the first line using tab separator
      else firstResult

    new CsvFile<'RowType>(finalResult, hasHeaders, readerFunc, ignoreErrors, stringArrayToRow, rowToStringArray, disposer, separators, quote)    

  /// [omit]
  private new (parsedFirstLine, hasHeaders, (readerFunc:Func<TextReader>), ignoreErrors, stringArrayToRow, rowToStringArray, disposer, separators, quote) as this =
    let {LineIterator = linesIterator; FirstLine = firstLine; ColumnCount = numberOfColumns; Headers = headers} = parsedFirstLine
    // Create sequence that is exposed as 'Data' - on the first read, finish
    // reading the opened reader; on future reads, get a new reader (and skip headers)
    let firstData = seq {
      use ls = linesIterator
      if not hasHeaders then yield firstLine
      while ls.MoveNext() do yield ls.Current }
    let nextData () = seq { 
      let reader = readerFunc.Invoke()
      let csv = CsvReader.readCsvFile reader separators quote
      yield! if hasHeaders then Seq.skip 1 csv else csv }
    let data = ReentrantEnumerable<_>(firstData, nextData)

    // Return data with parsed columns
    let data = seq {
      for row, lineNumber in data do
        if row.Length <> numberOfColumns then
          // Ignore rows with different number of columns when ignoreErrors is set to true
          if not ignoreErrors then
            let lineNumber = if hasHeaders then lineNumber else lineNumber + 1
            failwithf "Couldn't parse row %d according to schema: Expected %d columns, got %d" lineNumber numberOfColumns row.Length
        else
          // Always ignore empty rows
{'fraction_non_alphanumeric': 0.14339788249492566, 'fraction_numerical': 0.002633166931811948, 'mean_word_length': 2.3746760459089225, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1165197', 'n_tokens_mistral': 4893, 'n_tokens_neox': 4719, 'n_words': 1688}