<commit_before>define 'cs!xlform/csv', ->
  # The Csv class is only internally accessible, though easy
  # to instantiate through the `csv` function.
  #
  # It receives a parameter (and optional options) which
  # are stored for later use.
  class Csv
    constructor: (param, opts={})->
      if _isString param
        @string = param
        rows = csv.toArray @string
        @rows = arrayToObjects rows
        [@columns, @rowArray...] = rows
      else if _isArray param
        @rows = param
        @columns = do =>
          columns = []
          for row in @rows
            columns.push key  for own key of row when key not in columns
          columns
        @buildRowArray()
        @obj = param
      else if param
        @columns  = if _isArray param.columns then param.columns else []
        if param.rowObjects
          if @columns.length is 0
            # this may be slow, but it should get all the possible columns
            for row in param.rowObjects
              for key, val of row when key not of columns
                @columns.push key
          @rowArray = do =>
            for row in param.rowObjects
              row[c] for c in @columns
        else
          @rowArray = if _isArray param.rows then param.rows else []
        @kind     = param.kind if param.kind?
        @rows = do =>
          for row in @rowArray
            rowObj = {}
            for cell, i in row  when @columns[i]?
              rowObj[@columns[i]] = cell
            rowObj
      else
        @rows     = []
        @columns  = []
        @rowArray = []

    buildRowArray: ()->
      @rowArray = do =>
        for row in @rows
          for column in @columns
            row[column] || ""

    addRow: (r)->
      colsChanged = false
      for own key, val of r
        unless key in @columns
          colsChanged = true
          @columns.push key
      @rows.push r
      if colsChanged
        @buildRowArray()
      else
        @rowArray.push (r[column] for column in @columns)
      r

    toObjects: (opts={})->
      if @string
        return csv.toObjects @string, opts
      else if @rows
        @rows

    toObject: ()->
      out =
        columns: @columns
        rows: @rowArray
      out.kind = @kind  if @kind
      out

    toArrays: ()->
      out = [@columns]
      out.push row  for row in @rowArray when row.length > 0
      out

    toString: ()->
      headRow = (asCsvCellValue cell for cell in @columns).join(csv.settings.delimiter)
      headRow + "\n" + (for row in @rowArray
        (asCsvCellValue cell for cell in row).join csv.settings.delimiter).join("\n")

  csv = (param, opts)->
    if param instanceof Csv then param else new Csv param, opts

  _remove_extra_escape_chars = (ss)->
    ss.replace(/\\\\/g, '\\')

  asCsvCellValue = (cell)->
    if cell is undefined
      ""
    else if ///\W|\w|#{csv.settings.delimiter}///.test cell
      outstr = _remove_extra_escape_chars(cell)
      outstr = JSON.stringify("" + outstr)
      _remove_extra_escape_chars outstr
    else
      cell

  csv.fromStringArray = (outpArr, opts={})->
    outArr = for row in outpArr
      (asCsvCellValue cell for cell in row).join csv.settings.delimiter
    outArr.join "\n"

  csv.fromArray = (arr, opts={})->
    sort        = !!opts.sort
    headRow     = []
    for row in arr
      headRow.push key  for own key of row when -1 is headRow.indexOf key
    headRow.sort()  if sort
    outpArr = for row in arr
      asCsvCellValue row[col]  for col in headRow
    outpArr.unshift (asCsvCellValue col  for col in headRow)
    csv.fromStringArray outpArr, opts

  csv.toObjects = (csvString)->
    arrayToObjects csv.toArray csvString

  arrayToObjects = (arr)->
    [headRow, rows...] = arr
    for row in rows when !(row.length is 1 and row[0] is "")
      obj = {}
      for key, i in headRow
        obj[key] = row[i]
      obj

  csv.toObject = (csvString, opts)->
    arrayToObject csv.toArray(csvString), opts

  arrayToObject = (arr, opts={})->
    [headRow, rows...] = arr

    sortByKey         = opts.sortByKey
    includeSortByKey  = opts.includeSortByKey

    unless sortByKey
      sortByKey = headRow[0]

    sortByKeyI = headRow.indexOf sortByKey

    out = {}
    for row in rows when !(row.length is 1 and row[0] is "")
      obj = {}
      sbKeyVal = row[sortByKeyI]
      obj[key] = row[i]  for key, i in headRow when i isnt sortByKeyI
      obj[sortByKey] = sbKeyVal  if includeSortByKey
      out[sbKeyVal] = obj
    out

  removeTrailingNewlines = (str)-> str.replace(/(\n+)$/g, "")

  csv._parse_string = (c)->
    JSON.parse('"' + c.replace(/\\/g, '\\\\') + '"')

  # The `csv.toArray` function, pulled from this [stackoverflow comment](http://stackoverflow.com/questions/1293147/javascript-code-to-parse-csv-data)
  # will parse a delimited string into an array of
  # arrays. The default delimiter is the comma, but this<commit_msg>nothin to see here<commit_after>define 'cs!xlform/csv', ->
  # The Csv class is only internally accessible, though easy
  # to instantiate through the `csv` function.
  #
  # It receives a parameter (and optional options) which
  # are stored for later use.
  class Csv
    constructor: (param, opts={})->
      if _isString param
        @string = param
        rows = csv.toArray @string
        @rows = arrayToObjects rows
        [@columns, @rowArray...] = rows
      else if _isArray param
        @rows = param
        @columns = do =>
          columns = []
          for row in @rows
            columns.push key  for own key of row when key not in columns
          columns
        @buildRowArray()
        @obj = param
      else if param
        @columns  = if _isArray param.columns then param.columns else []
        if param.rowObjects
          if @columns.length is 0
            # this may be slow, but it should get all the possible columns
            for row in param.rowObjects
              for key, val of row when key not of columns
                @columns.push key
          @rowArray = do =>
            for row in param.rowObjects
              row[c] for c in @columns
        else
          @rowArray = if _isArray param.rows then param.rows else []
        @kind     = param.kind if param.kind?
        @rows = do =>
          for row in @rowArray
            rowObj = {}
            for cell, i in row  when @columns[i]?
              rowObj[@columns[i]] = cell
            rowObj
      else
        @rows     = []
        @columns  = []
        @rowArray = []

    buildRowArray: ()->
      @rowArray = do =>
        for row in @rows
          for column in @columns
            row[column] || ""

    addRow: (r)->
      colsChanged = false
      for own key, val of r
        unless key in @columns
          colsChanged = true
          @columns.push key
      @rows.push r
      if colsChanged
        @buildRowArray()
      else
        @rowArray.push (r[column] for column in @columns)
      r

    toObjects: (opts={})->
      if @string
        return csv.toObjects @string, opts
      else if @rows
        @rows

    toObject: ()->
      out =
        columns: @columns
        rows: @rowArray
      out.kind = @kind  if @kind
      out

    toArrays: ()->
      out = [@columns]
      out.push row  for row in @rowArray when row.length > 0
      out

    toString: ()->
      headRow = (asCsvCellValue cell for cell in @columns).join(csv.settings.delimiter)
      headRow + "\n" + (for row in @rowArray
        (asCsvCellValue cell for cell in row).join csv.settings.delimiter).join("\n")

  csv = (param, opts)->
    if param instanceof Csv then param else new Csv param, opts

  _remove_extra_escape_chars = (ss)->
    ss.replace(/\\\\/g, '\\')

  asCsvCellValue = (cell)->
    if cell is undefined
      ""
    else if ///\W|\w|#{csv.settings.delimiter}///.test cell
      outstr = JSON.stringify("" + cell)
      _remove_extra_escape_chars outstr
    else
      cell

  csv.fromStringArray = (outpArr, opts={})->
    outArr = for row in outpArr
      (asCsvCellValue cell for cell in row).join csv.settings.delimiter
    outArr.join "\n"

  csv.fromArray = (arr, opts={})->
    sort        = !!opts.sort
    headRow     = []
    for row in arr
      headRow.push key  for own key of row when -1 is headRow.indexOf key
    headRow.sort()  if sort
    outpArr = for row in arr
      asCsvCellValue row[col]  for col in headRow
    outpArr.unshift (asCsvCellValue col  for col in headRow)
    csv.fromStringArray outpArr, opts

  csv.toObjects = (csvString)->
    arrayToObjects csv.toArray csvString

  arrayToObjects = (arr)->
    [headRow, rows...] = arr
    for row in rows when !(row.length is 1 and row[0] is "")
      obj = {}
      for key, i in headRow
        obj[key] = row[i]
      obj

  csv.toObject = (csvString, opts)->
    arrayToObject csv.toArray(csvString), opts

  arrayToObject = (arr, opts={})->
    [headRow, rows...] = arr

    sortByKey         = opts.sortByKey
    includeSortByKey  = opts.includeSortByKey

    unless sortByKey
      sortByKey = headRow[0]

    sortByKeyI = headRow.indexOf sortByKey

    out = {}
    for row in rows when !(row.length is 1 and row[0] is "")
      obj = {}
      sbKeyVal = row[sortByKeyI]
      obj[key] = row[i]  for key, i in headRow when i isnt sortByKeyI
      obj[sortByKey] = sbKeyVal  if includeSortByKey
      out[sbKeyVal] = obj
    out

  removeTrailingNewlines = (str)-> str.replace(/(\n+)$/g, "")

  csv._parse_string = (c)->
    JSON.parse('"' + c.replace(/\\/g, '\\\\').replace(/\\\\"/g, '\\"') + '"')

  # The `csv.toArray` function, pulled from this [stackoverflow comment](http://stackoverflow.com/questions/1293147/javascript-code-to-parse-csv-data)
  # will parse a delimited string into an array of
  # arrays. The default delimiter is the comma, but this
{'fraction_non_alphanumeric': 0.10903331289597384, 'fraction_numerical': 0.0030656039239730227, 'mean_word_length': 1.973868125189912, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325305', 'n_tokens_mistral': 3087, 'n_tokens_neox': 2978, 'n_words': 969}