<commit_before>'use strict';

var Promise = require('bluebird');

module.exports = {
  up: function (queryInterface, Sequelize) {
    var fs = require('fs');
    var parse = require('csv-parse');
    var parser = parse({columns: true, skip_empty_lines: true});
    var inserts = [];
    var completed = 0;
    var lastNotice = 0;

    function notify(force) {
      if (!force && Date.now()-lastNotice < 5000) return;
      lastNotice = Date.now();
      console.log('waiting ' + (inserts.length-completed) + "/"+inserts.length);
    }

    var stream = fs.createReadStream(__dirname + '/../../data/locations.csv')
      .pipe(parser)
      .on('readable', function () {
        var record, i;
        while (record = parser.read()) {
          var zoneId = record['UTMNameFul'];
          var fields = {
            locationNameBg: record['Name_bg_naseleno_myasto'],
            locationNameEn: record['Name_en_naseleno_myasto'],
            locationAreaBg: record['NAME_Obshtina'],
            locationAreaEn: record['L_NAME_Obshtina'],
            locationTypeBg: record['Descr_bg_naseleno_myasto'],
            locationTypeEn: record['Descr_en_naseleno_myasto']
          };
          inserts.push(queryInterface.bulkUpdate('Zones', fields, {id: zoneId}).then(function () {
            completed++;
            notify();
          }));
        }
      });

    return new Promise(function (resolve, reject) {

      stream
        .on('error', function (err) {
          console.error('error', err);
          reject(err);
        })
        .on('end', function () {
          notify(true);
          resolve(Promise.all(inserts));
        });

    });
  },

  down: function (queryInterface, Sequelize) {
    return Promise.resolve(true);
  }
};<commit_msg>Update zone location seed to latest schema<commit_after>'use strict';

var _ = require('lodash');
var Promise = require('bluebird');

module.exports = {
  up: function (queryInterface, Sequelize) {
    var fs = require('fs');
    var parse = require('csv-parse');
    var parser = parse({columns: true, skip_empty_lines: true});
    var inserts = [];
    var completed = 0;
    var lastNotice = 0;
    var cache = {};

    function notify(force) {
      if (!force && Date.now() - lastNotice < 5000) return;
      lastNotice = Date.now();
      console.log('waiting ' + (inserts.length - completed) + "/" + inserts.length);
    }

    var stream = fs.createReadStream(__dirname + '/../../data/locations.csv')
      .pipe(parser)
      .on('readable', function () {
        var record, i;
        while (record = parser.read()) {
          var zoneId = record['UTMNameFul'];
          var fields = {
            nameBg: record['Name_bg_naseleno_myasto'],
            nameEn: record['Name_en_naseleno_myasto'],
            areaBg: record['NAME_Obshtina'],
            areaEn: record['L_NAME_Obshtina'],
            typeBg: record['Descr_bg_naseleno_myasto'],
            typeEn: record['Descr_en_naseleno_myasto']
          };
          (function (zoneId, fields) {
            var key = _.reduce(fields, function (sum, val) {
              return sum + ' ' + val;
            }, '');
            inserts.push(Promise.resolve(fields)
              .then(function (fields) {
                if (key in cache)
                  return cache[key];
                return cache[key] = queryInterface.rawSelect('Locations', {
                    attributes: ['id'],
                    where: fields
                  }, 'id')

                  .then(function (id) {
                    if (id !== null)
                      return id;

                    var record = _.extend({
                      createdAt: new Date(),
                      updatedAt: new Date()
                    }, fields);
                    return queryInterface.bulkInsert('Locations', [record])
                      .then(function () {
                        return queryInterface.rawSelect('Locations', {
                          attributes: ['id'],
                          where: fields
                        }, 'id');
                      });
                  });
              })
              .then(function (locationId) {
                return queryInterface.bulkUpdate('Zones', {
                  locationId: locationId
                }, {id: zoneId});
              })
              .then(function () {
                completed++;
                notify();
              })
            );
          })(zoneId, fields);
        }
      });

    return new Promise(function (resolve, reject) {

      stream
        .on('error', function (err) {
          console.error('error', err);
          reject(err);
        })
        .on('end', function () {
          notify(true);
          Promise.all(inserts).then(resolve, reject);
        });

    });
  },

  down: function (queryInterface, Sequelize) {
    return Promise.resolve(true);
  }
};
{'fraction_non_alphanumeric': 0.13767081378747706, 'fraction_numerical': 0.0024474811339995923, 'mean_word_length': 1.7004405286343611, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7144990', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1318, 'n_words': 277}