<commit_before>var SourceModel = require('../../models/source')
  , exec = require('child_process').exec
  , config = require('../../config.json')
  , fs = require('fs-extra');

exports.process = function(source, callback) {
  callback(null, source);
  var child = require('child_process').fork('api/sources/processor.js');
  child.send({operation:'process', sourceId: source.id});
}

exports.getTile = function(source, z, x, y, params, callback) {
  console.log('get tile ' + z + '/' + x + '/' + y + '.png for source ' + source.name);

  var tile = config.server.sourceDirectory.path + "/" + source._id + "/tiles/" + z + '/' + x + '/' + y + '.png';

  if (fs.existsSync(tile)) {
    var stream = fs.createReadStream(tile);
    var tileSize = 0;
    stream.on('data', function(chunk) {
      tileSize += chunk.length;
    });
    stream.on('end', function() {
      SourceModel.updateSourceAverageSize(source, tileSize, function(err) {
      });
    });
    callback(null, stream);
  } else {
    callback();
  }
}

exports.getData = function(source, callback) {
  callback(null);
}

exports.processSource = function(source, callback) {
  console.log('running ' + 'mb-util ' + source.filePath + " .");
  source.status = "Extracting MBTiles";
  source.save();
  callback(null, source);
  var python = exec(
    'mb-util ' + source.filePath + " " + config.server.sourceDirectory.path + "/" + source._id + "/tiles",
   function(error, stdout, stderr) {
     source.status = "Complete";
     source.complete = true;
     source.save();
     console.log('done running ' +   'mb-util ' + source.filePath + " " + config.server.sourceDirectory.path + "/" + source._id + "/tiles");
     callback();
   });
}<commit_msg>fixing mbtile source creation<commit_after>var SourceModel = require('../../models/source')
  , exec = require('child_process').exec
  , config = require('../../config.json')
  , fs = require('fs-extra');

exports.process = function(source, callback) {
  callback(null, source);
  var child = require('child_process').fork('api/sources/processor.js');
  child.send({operation:'process', sourceId: source.id});
}

exports.getTile = function(source, z, x, y, params, callback) {
  console.log('get tile ' + z + '/' + x + '/' + y + '.png for source ' + source.name);

  var tile = config.server.sourceDirectory.path + "/" + source._id + "/tiles/" + z + '/' + x + '/' + y + '.png';

  if (fs.existsSync(tile)) {
    var stream = fs.createReadStream(tile);
    var tileSize = 0;
    stream.on('data', function(chunk) {
      tileSize += chunk.length;
    });
    stream.on('end', function() {
      SourceModel.updateSourceAverageSize(source, tileSize, function(err) {
      });
    });
    callback(null, stream);
  } else {
    callback();
  }
}

exports.getData = function(source, callback) {
  callback(null);
}

exports.processSource = function(source, callback) {
  console.log('running ' + 'mb-util ' + source.filePath + " " + config.server.sourceDirectory.path + "/" + source._id + "/tiles");
  source.status = "Extracting MBTiles";
  source.save(function() {
    var python = exec(
      'mb-util ' + source.filePath + " " + config.server.sourceDirectory.path + "/" + source._id + "/tiles",
     function(error, stdout, stderr) {
       source.status = "Complete";
       source.complete = true;
       source.save(function() {
         console.log('done running ' +   'mb-util ' + source.filePath + " " + config.server.sourceDirectory.path + "/" + source._id + "/tiles");
         callback();
       });
     });
   });
}
{'fraction_non_alphanumeric': 0.18976244343891402, 'fraction_numerical': 0.0005656108597285068, 'mean_word_length': 3.195729537366548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189109', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1121, 'n_words': 216}