<commit_before>'use strict'

var fs = require('fs')
var path = require('path')
var parse = require('csv-parse')
var parser = parse({
  columns: true,
  skip_empty_lines: true,
  delimiter: ';'
})
var inserts = []
var Promise = require('bluebird')
var completed = 0
var lastNotice = 0

function notify(force) {
  if (!force && Date.now() - lastNotice < 5000) return
  lastNotice = Date.now()
  console.log('processed ' + completed + '/' + inserts.length)
}

function importRecord(queryInterface, record) {
  return queryInterface.bulkInsert('Species', [{
    type: record.type.trim() || null,
    labelLa: record.la.trim() || null,
    labelBg: (record.bg || '').trim() || null,
    labelEn: (record.en || '').trim() || null,
    createdAt: new Date(),
    updatedAt: new Date()
  }])
}

module.exports = {
  up: function (queryInterface, Sequelize, next) {
    var stream = fs.createReadStream(path.join(__dirname, '..', '..', 'data', 'species_invertebrates.csv'))
      .pipe(parser)
      .on('readable', function () {<commit_msg>fix standard issues<commit_after>'use strict'

var fs = require('fs')
var path = require('path')
var parse = require('csv-parse')
var parser = parse({
  columns: true,
  skip_empty_lines: true,
  delimiter: ';'
})
var inserts = []
var Promise = require('bluebird')
var completed = 0
var lastNotice = 0

function notify (force) {
  if (!force && Date.now() - lastNotice < 5000) return
  lastNotice = Date.now()
  console.log('processed ' + completed + '/' + inserts.length)
}

function importRecord (queryInterface, record) {
  return queryInterface.bulkInsert('Species', [{
    type: record.type.trim() || null,
    labelLa: record.la.trim() || null,
    labelBg: (record.bg || '').trim() || null,
    labelEn: (record.en || '').trim() || null,
    createdAt: new Date(),
    updatedAt: new Date()
  }])
}

module.exports = {
  up: function (queryInterface, Sequelize, next) {
    var stream = fs.createReadStream(path.join(__dirname, '..', '..', 'data', 'species_invertebrates.csv'))
      .pipe(parser)
      .on('readable', function () {
{'fraction_non_alphanumeric': 0.1776061776061776, 'fraction_numerical': 0.005791505791505791, 'mean_word_length': 3.900709219858156, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3828369', 'n_tokens_mistral': 703, 'n_tokens_neox': 668, 'n_words': 168}