<commit_before>import _ from 'lodash'
import stringify from 'csv-stringify'
import Promise from 'bluebird'
import iconv from 'iconv-lite'
import Excel from 'exceljs'

const fs = Promise.promisifyAll(require('fs'))

const DEBUG = false
const debugLog = DEBUG ? console.log : _.noop

export default class Writer {
  constructor (options) {
    this.options = options

    debugLog('WRITER::options:', JSON.stringify(options))

    this.options.availableFormats = ['xlsx', 'csv']
    this.options.defaultEncoding = 'utf8'

    if (options.availableFormats.indexOf(options.exportFormat) < 0)
      throw new Error(`Unsupported file type: ${options.exportFormat}, `
        + `alowed formats are ${options.availableFormats.toString()}`)

    if (this.options.outputFile) {
      debugLog('WRITER::stream file %s', options.outputFile)
      this.outputStream = fs.createWriteStream(options.outputFile)
    } else {
      debugLog('WRITER::stream stdout')
      this.outputStream = process.stdout
    }

    // if we use xlsx export - create workbook first
    if (options.exportFormat === 'xlsx')
      this.options.workbookOpts = {
        stream: this.outputStream,
        useStyles: true,
        useSharedStrings: true,
      }

    this.workbook = new Excel.stream.xlsx.WorkbookWriter(options.workbookOpts)
    this.worksheet = this.workbook.addWorksheet('Worksheet1')
  }

  encode (string) {
    if (this.options.encoding === this.options.defaultEncoding)
      return string

    if (!iconv.encodingExists(this.options.encoding))
      throw new Error(`Encoding does not exist: ${this.options.encoding}`)

    return iconv.encode(string, this.options.encoding)
  }

  // create header
  setHeader (header) {
    debugLog('WRITER::writing header of len %d', header.length)

    if (this.options.exportFormat === 'xlsx')
      return this._writeXlsxHeader(header)
    return this._writeCsvRows([header])
  }

  write (rows) {
    debugLog('WRITER::writing rows len: %d', rows.length)

    if (this.options.exportFormat === 'xlsx')
      return this._writeXlsxRows(rows)
    return this._writeCsvRows(rows)
  }

  _writeXlsxRows (rows) {
    return Promise.map(rows, (row) => {
      // clean row from undefined and empty strings
      const cleanedRow = _.map(row, item => (_.isNil(item) ? null : item))

      this.worksheet.addRow(cleanedRow).commit()
    }, { concurrency: 1 })
  }

  _writeXlsxHeader (header) {
    this.worksheet.columns = header.map(name => ({ header: name }))
    return Promise.resolve()
  }

  _writeCsvRows (data) {
    const opts = {
      delimiter: this.options.csvDelimiter,
    }

    return new Promise((resolve, reject) =>
      stringify(data, opts, (err, string) => {
        if (err)
          return reject(err)

        this.outputStream.write(this.encode(string))
        return resolve()<commit_msg>fix(export): Fix serializing of boolean values in xlsx format<commit_after>import _ from 'lodash'
import stringify from 'csv-stringify'
import Promise from 'bluebird'
import iconv from 'iconv-lite'
import Excel from 'exceljs'

const fs = Promise.promisifyAll(require('fs'))

const DEBUG = false
const debugLog = DEBUG ? console.log : _.noop

export default class Writer {
  constructor (options) {
    this.options = options

    debugLog('WRITER::options:', JSON.stringify(options))

    this.options.availableFormats = ['xlsx', 'csv']
    this.options.defaultEncoding = 'utf8'

    if (options.availableFormats.indexOf(options.exportFormat) < 0)
      throw new Error(`Unsupported file type: ${options.exportFormat}, `
        + `alowed formats are ${options.availableFormats.toString()}`)

    if (this.options.outputFile) {
      debugLog('WRITER::stream file %s', options.outputFile)
      this.outputStream = fs.createWriteStream(options.outputFile)
    } else {
      debugLog('WRITER::stream stdout')
      this.outputStream = process.stdout
    }

    // if we use xlsx export - create workbook first
    if (options.exportFormat === 'xlsx')
      this.options.workbookOpts = {
        stream: this.outputStream,
        useStyles: true,
        useSharedStrings: true,
      }

    this.workbook = new Excel.stream.xlsx.WorkbookWriter(options.workbookOpts)
    this.worksheet = this.workbook.addWorksheet('Worksheet1')
  }

  encode (string) {
    if (this.options.encoding === this.options.defaultEncoding)
      return string

    if (!iconv.encodingExists(this.options.encoding))
      throw new Error(`Encoding does not exist: ${this.options.encoding}`)

    return iconv.encode(string, this.options.encoding)
  }

  // create header
  setHeader (header) {
    debugLog('WRITER::writing header of len %d', header.length)

    if (this.options.exportFormat === 'xlsx')
      return this._writeXlsxHeader(header)
    return this._writeCsvRows([header])
  }

  write (rows) {
    debugLog('WRITER::writing rows len: %d', rows.length)

    if (this.options.exportFormat === 'xlsx')
      return this._writeXlsxRows(rows)
    return this._writeCsvRows(rows)
  }

  _writeXlsxRows (rows) {
    return Promise.map(rows, (row) => {
      // clean row from undefined and empty strings
      const cleanedRow = _.map(row, (item) => {
        if (_.isNil(item))
          return null
        else if (_.isBoolean(item))
          return (item ? 1 : '')
        return item
      })

      this.worksheet.addRow(cleanedRow).commit()
    }, { concurrency: 1 })
  }

  _writeXlsxHeader (header) {
    this.worksheet.columns = header.map(name => ({ header: name }))
    return Promise.resolve()
  }

  _writeCsvRows (data) {
    const opts = {
      delimiter: this.options.csvDelimiter,
    }

    return new Promise((resolve, reject) =>
      stringify(data, opts, (err, string) => {
        if (err)
          return reject(err)

        this.outputStream.write(this.encode(string))
        return resolve()
{'fraction_non_alphanumeric': 0.12512820512820513, 'fraction_numerical': 0.0015384615384615385, 'mean_word_length': 3.1852646638054365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2047644', 'n_tokens_mistral': 1822, 'n_tokens_neox': 1776, 'n_words': 406}