<commit_before>
fs = require 'fs'
should = require 'should'
produce = require 'produce'
parse = if process.env.CSV_COV then require '../lib-cov' else require '../src'

describe 'quote', ->
  
  it 'with default',  (next) ->
    parse """
    20322051544,"1979.0",8.8017226E7,"ABC,45","2000-01-01"
    28392898392,1974.0,"8.8392926E7",DEF,23,2050-11-27
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','1979.0','8.8017226E7','ABC,45','2000-01-01' ]
        [ '28392898392','1974.0','8.8392926E7','DEF','23','2050-11-27' ]
      ]
      next()
    
  it 'with fields containing delimiters', (next) ->
    parse """
    20322051544,",1979.0,8.8017226E7,ABC,45,2000-01-01"
    28392898392,1974.0,8.8392926E7,DEF,23,2050-11-27
    "28392898392,1974.0","8.8392926E7","DEF,23,2050-11-27,"
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544',',1979.0,8.8017226E7,ABC,45,2000-01-01' ]
        [ '28392898392','1974.0','8.8392926E7','DEF','23','2050-11-27' ]
        [ '28392898392,1974.0','8.8392926E7','DEF,23,2050-11-27,' ]
      ]
      next()
    
  it 'with fields containing quotes', (next) ->
    parse """
    20322051544,"1979.0",8.801"7226E7,ABC,45,2000-01-01
    28392898392,1974.0,8.8392926E7,DEF,2"3,2050-11-27
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','1979.0','8.801"7226E7','ABC','45','2000-01-01' ]
        [ '28392898392','1974.0','8.8392926E7','DEF','2"3','2050-11-27' ]
      ]
      next()
    
  it 'empty value', (next) ->
    parse """
    20322051544,"",8.8017226E7,45,""
    "",1974,8.8392926E7,"",""
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','','8.8017226E7','45','' ]
        [ '','1974','8.8392926E7','','' ]
      ]
      next()
    
  it 'values containing quotes and double quotes escape', (next) ->
    parse """
    20322051544,\"\"\"\",8.8017226E7,45,\"\"\"ok\"\"\"
    "",1974,8.8392926E7,"",""
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','"','8.8017226E7',45,'"ok"' ]
        [ '','1974','8.8392926E7','','' ]
      ]
      next()
    
  it 'line breaks inside quotes', (next) ->
    parse """
    20322051544,"
    ",8.8017226E7,45,"
    ok
    "
    "
    ",1974,8.8392926E7,"","
    "
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','\n',',8.8017226E7',45,'\nok\n' ]
        [ '\n','1974','8.8392926E7','','\n' ]
      ]
      next()

describe 'quote error', ->
  
  it 'when unclosed', (next) ->
    parse """<commit_msg>Fix quote example<commit_after>
fs = require 'fs'
should = require 'should'
produce = require 'produce'
parse = if process.env.CSV_COV then require '../lib-cov' else require '../src'

describe 'quote', ->
  
  it 'with default',  (next) ->
    parse """
    20322051544,"1979.0",8.8017226E7,"ABC,45","2000-01-01"
    28392898392,1974.0,"8.8392926E7",DEF,23,2050-11-27
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','1979.0','8.8017226E7','ABC,45','2000-01-01' ]
        [ '28392898392','1974.0','8.8392926E7','DEF','23','2050-11-27' ]
      ]
      next()
    
  it 'with fields containing delimiters', (next) ->
    parse """
    20322051544,",1979.0,8.8017226E7,ABC,45,2000-01-01"
    28392898392,1974.0,8.8392926E7,DEF,23,2050-11-27
    "28392898392,1974.0","8.8392926E7","DEF,23,2050-11-27,"
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544',',1979.0,8.8017226E7,ABC,45,2000-01-01' ]
        [ '28392898392','1974.0','8.8392926E7','DEF','23','2050-11-27' ]
        [ '28392898392,1974.0','8.8392926E7','DEF,23,2050-11-27,' ]
      ]
      next()
    
  it 'with fields containing quotes', (next) ->
    parse """
    20322051544,"1979.0",8.801"7226E7,ABC,45,2000-01-01
    28392898392,1974.0,8.8392926E7,DEF,2"3,2050-11-27
    """, relax: true, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','1979.0','8.801"7226E7','ABC','45','2000-01-01' ]
        [ '28392898392','1974.0','8.8392926E7','DEF','2"3','2050-11-27' ]
      ]
      next()
    
  it 'empty value', (next) ->
    parse """
    20322051544,"",8.8017226E7,45,""
    "",1974,8.8392926E7,"",""
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','','8.8017226E7','45','' ]
        [ '','1974','8.8392926E7','','' ]
      ]
      next()
    
  it 'values containing quotes and double quotes escape', (next) ->
    parse """
    20322051544,\"\"\"\",8.8017226E7,45,\"\"\"ok\"\"\"
    "",1974,8.8392926E7,"",""
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','"','8.8017226E7',45,'"ok"' ]
        [ '','1974','8.8392926E7','','' ]
      ]
      next()
    
  it 'line breaks inside quotes', (next) ->
    parse """
    20322051544,"
    ",8.8017226E7,45,"
    ok
    "
    "
    ",1974,8.8392926E7,"","
    "
    """, (err, data) ->
      return next err if err
      data.should.eql [
        [ '20322051544','\n','8.8017226E7',45,'\nok\n' ]
        [ '\n','1974','8.8392926E7','','\n' ]
      ]
      next()

describe 'quote error', ->
  
  it 'when unclosed', (next) ->
    parse """
{'fraction_non_alphanumeric': 0.22806691449814126, 'fraction_numerical': 0.26022304832713755, 'mean_word_length': 2.5146962769431744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2594820', 'n_tokens_mistral': 2992, 'n_tokens_neox': 2275, 'n_words': 308}