<commit_before>var tilelive = require('..');
var url = require('url');
var fs = require('fs');
var Scheme = require('./scheme');

module.exports = CopyTask;
require('util').inherits(CopyTask, process.EventEmitter);
function CopyTask(from, to, scheme, job) {
    this.from = typeof from === 'string' ? url.parse(from) : from;
    this.to = typeof to === 'string' ? url.parse(to) : to;
    this.scheme = scheme;
    this.started = null;
    this.ended = null;
    this.job = job;
    this.formats = [];
    this.initialize(from, to);
}

CopyTask.prototype.initialize = function(from, to) {
    var task = this;

    this.stats = this.scheme.stats;
    this.emit('progress', this.stats.snapshot());

    // Make sure this is non-enumerable;
    Object.defineProperty(this.scheme, 'task', { value: this });

    // State persistence
    console.warn('Persisting state in ' + this.job + ' every minute');

    process.on('uncaughtException', function(err) {
        if (err.stack) console.warn('\n' + err.stack);
        else console.warn('\n' + err.message);
        console.warn('Trying to exit cleanly...');
        task.pause(function(err) {
            if (err) throw err;
            console.warn('\nWrote state to ' + task.job);
        });
    });

    if ((process.platform !== 'win32')) {
        process.on('SIGINT', function() {
            task.pause(function(err) {
                if (err) throw err;
                console.warn('\nWrote state to ' + task.job);
            });
        });
    }
};

CopyTask.prototype.toJSON = function() {
    return {
        started: this.started,
        ended: this.ended,
        from: this.from,
        to: this.to,
        scheme: this.scheme
    };
};

CopyTask.prototype.finished = function() {
    var task = this;
    this.pause(function() {
        task.emit('finished');
    });
};

CopyTask.prototype.pause = function(callback) {
    var task = this;

    this.scheme.once('paused', function() {
        clearInterval(task.progressInterval);
        clearInterval(task.stateInterval);
        task.emit('progress', task.stats.snapshot());

        task.sink.stopWriting(function(err) {
            task.ended = Date.now();
            task.emit('progress', task.stats.snapshot());
            if (err) return callback(err);
            fs.writeFile(task.job, JSON.stringify(task), function(err) {
                if (err) return callback(err);
                task.sink.close(function(err) {
                    if (err) return callback(err);
                    task.source.close(callback);
                });
            });
        });
    });

    this.scheme.pause();
};

CopyTask.prototype.start = function(callback) {
    var task = this
    if (!this.started) this.started = Date.now();

    this.progressInterval = setInterval(function() {
        task.emit('progress', task.stats.snapshot());
    }, 1000);

    this.stateInterval = setInterval(function() {
        fs.writeFile(task.job, JSON.stringify(task));
    }, 60000);

    tilelive.load(task.from, function(err, source) {
        if (err) return callback(err);
        task.source = source;
        // @TODO - should a source's info provide target formats?
        if (task.source._mml && task.source._mml.tiles) {
            if (typeof task.source.getTile !== 'function') throw new Error('source must implement the tilesource interface');
            task.formats.push('tile');
        }
        if (task.source._mml && task.source._mml.grids) {
            if (typeof task.source.getGrid !== 'function') throw new Error('source must implement the tilesource interface');
            task.formats.push('grid');
        }
        if (task.formats.length === 0) {
            throw new Error("source must define some target formats");
        }
        tilelive.load(task.to, function(err, sink) {
            if (err) return callback(err);
            task.sink = sink;
            task.sink.startWriting(function(err) {
                if (err) return callback(err);
                task.scheme.start();
                callback(null);
            });
        });
    });
};

CopyTask.prototype.render = function(tile,type) {
    var get = type === 'grid' ? 'getGrid' : 'getTile';
    var put = type === 'grid' ? 'putGrid' : 'putTile';
    var putDupe = type === 'grid' ? 'putDuplicateGrid' : 'putDuplicateTile';
    var task = this;
    if (tile.key !== false) {
        task.sink[putDupe](tile.z, tile.x, tile.y, tile.key, function(err) {
            if (err) {
                task.emit('error', err, tile);
                task.scheme.error(tile);<commit_msg>Determine that grids are present using getInfo and template key. Refs #44.<commit_after>var tilelive = require('..');
var url = require('url');
var fs = require('fs');
var Scheme = require('./scheme');

module.exports = CopyTask;
require('util').inherits(CopyTask, process.EventEmitter);
function CopyTask(from, to, scheme, job) {
    this.from = typeof from === 'string' ? url.parse(from) : from;
    this.to = typeof to === 'string' ? url.parse(to) : to;
    this.scheme = scheme;
    this.started = null;
    this.ended = null;
    this.job = job;
    this.formats = [];
    this.initialize(from, to);
}

CopyTask.prototype.initialize = function(from, to) {
    var task = this;

    this.stats = this.scheme.stats;
    this.emit('progress', this.stats.snapshot());

    // Make sure this is non-enumerable;
    Object.defineProperty(this.scheme, 'task', { value: this });

    // State persistence
    console.warn('Persisting state in ' + this.job + ' every minute');

    process.on('uncaughtException', function(err) {
        if (err.stack) console.warn('\n' + err.stack);
        else console.warn('\n' + err.message);
        console.warn('Trying to exit cleanly...');
        task.pause(function(err) {
            if (err) throw err;
            console.warn('\nWrote state to ' + task.job);
        });
    });

    if ((process.platform !== 'win32')) {
        process.on('SIGINT', function() {
            task.pause(function(err) {
                if (err) throw err;
                console.warn('\nWrote state to ' + task.job);
            });
        });
    }
};

CopyTask.prototype.toJSON = function() {
    return {
        started: this.started,
        ended: this.ended,
        from: this.from,
        to: this.to,
        scheme: this.scheme
    };
};

CopyTask.prototype.finished = function() {
    var task = this;
    this.pause(function() {
        task.emit('finished');
    });
};

CopyTask.prototype.pause = function(callback) {
    var task = this;

    this.scheme.once('paused', function() {
        clearInterval(task.progressInterval);
        clearInterval(task.stateInterval);
        task.emit('progress', task.stats.snapshot());

        task.sink.stopWriting(function(err) {
            task.ended = Date.now();
            task.emit('progress', task.stats.snapshot());
            if (err) return callback(err);
            fs.writeFile(task.job, JSON.stringify(task), function(err) {
                if (err) return callback(err);
                task.sink.close(function(err) {
                    if (err) return callback(err);
                    task.source.close(callback);
                });
            });
        });
    });

    this.scheme.pause();
};

CopyTask.prototype.start = function(callback) {
    var task = this
    if (!this.started) this.started = Date.now();

    this.progressInterval = setInterval(function() {
        task.emit('progress', task.stats.snapshot());
    }, 1000);

    this.stateInterval = setInterval(function() {
        fs.writeFile(task.job, JSON.stringify(task));
    }, 60000);

    tilelive.load(task.from, function(err, source) {
        if (err) return callback(err);
        task.source = source;
        source.getInfo(function(err, info) {
            if (err) return callback(err);
            // @TODO the tilelive API currently has no standard way
            // of determining whether a tilesource has tiles, grids
            // or both. See issue #44.
            task.formats.push('tile');

            // Use template as an indicator that grids are present.
            if (info.template) task.formats.push('grid');

            tilelive.load(task.to, function(err, sink) {
                if (err) return callback(err);
                task.sink = sink;
                task.sink.startWriting(function(err) {
                    if (err) return callback(err);
                    task.scheme.start();
                    callback(null);
                });
            });
        });
    });
};

CopyTask.prototype.render = function(tile,type) {
    var get = type === 'grid' ? 'getGrid' : 'getTile';
    var put = type === 'grid' ? 'putGrid' : 'putTile';
    var putDupe = type === 'grid' ? 'putDuplicateGrid' : 'putDuplicateTile';
    var task = this;
    if (tile.key !== false) {
        task.sink[putDupe](tile.z, tile.x, tile.y, tile.key, function(err) {
            if (err) {
                task.emit('error', err, tile);
                task.scheme.error(tile);
{'fraction_non_alphanumeric': 0.15291395835628513, 'fraction_numerical': 0.0029745510631265836, 'mean_word_length': 2.153178186870441, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2589648', 'n_tokens_mistral': 2714, 'n_tokens_neox': 2632, 'n_words': 586}