<commit_before>var test = require('tape');
var fs = require('fs');
var stream = require('stream');
var tmp = require('os').tmpdir();
var path = require('path');
var exec = require('child_process').exec;
var tilelive = require('../');
var util = require('util');
var concat = require('concat-stream');
var MBTiles = require('mbtiles');
//register protocols
MBTiles.registerProtocols(tilelive);
var crypto = require('crypto');
var Timedsource = require('./timedsource');
var out = [];

var s3url = 's3://tilestream-tilesets-development/carol-staging/mapbox-tile-copy/{z}/{x}/{y}.png';

test('copy usage', function(t) {
    exec(__dirname + '/../bin/tilelive-copy', function(err, stdout, stderr) {
        t.equal(1, err.code, 'exit 1');
        t.ok(/Usage:/.test(stdout), 'shows usage');
        t.end();
    });
});

test('copy copies', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('286/') !== -1, '286');
        t.end();
    });
});

// test('copy copies to s3', function(t) {
//     exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles ' + s3url, function(err, stdout, stderr) {
//         t.ifError(err, 'no errors');
//         t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
//         t.end();
//     });
// });

test('copy min/max', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_minmax.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --minzoom=1 --maxzoom=2 ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('21/') !== -1, '21');
        t.end();
    });
});

test('copy bounds', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_bounds.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --bounds=-180,-85,0,0 ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('59') !== -1, '59');
        t.end();
    });
});

test('copy list', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_list.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --scheme=list --list=' + __dirname + '/fixtures/filescheme.flat ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('77') !== -1, '77');
        t.end();
    });
});

test('copy streams', function(t) {
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles', {maxBuffer:5e6}, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('JSONBREAKFASTTIME\n') === 0);
        t.equal(stdout.length, 647001);
        t.end();
    });
});

test('copy part zero', function(t) {
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles --part 0 --parts 10', function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.split('\n').length < 287, 'does not render all tiles');
        t.end();
    });
});

test('copy timeout', function(t) {
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles --timeout=1', function(err, stdout, stderr) {
        t.ok(err);
        t.ok(/Copy operation timed out/.test(stderr));
        t.end();
    });
});

test('copy slow', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_slow.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --slow=20 ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.ok((/\[slow tile\] (get|put) \d+\/\d+\/\d+ \d+ms/).test(stderr), 'logs slow tiles to stderr');
        t.end();
    });
});

test('tilelive.copy', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy.mbtiles');
    var options = {
        progress: report
    };

    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.end();
    });
});

test('tilelive.copy: concurrency', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_concurrency.mbtiles');
    var options = {
        progress: report,
        concurrency: 10
    };

    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.end();
    });
});

test('tilelive.copy: onslow', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_concurrency.mbtiles');
    var log = [];
    var options = {
        slow: 20,
        onslow: function(method, z, x, y, time) {
            t.equal(/^(get|put)$/.test(method), true);
            t.equal(typeof z, 'number');
            t.equal(typeof x, 'number');
            t.equal(typeof y, 'number');
            t.equal(typeof time, 'number');
            log.push({method:method, z:z, x:x, y:y, time:time});
        },
        progress: report,
        concurrency: 10
    };

    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.equal(log.length > 0, true, 'calls onslow callback');
        tilelive.stream.slowTime = 60e3;
        t.end();
    });
});

test('tilelive copy: list', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_list.mbtiles');
    var list = __dirname + '/fixtures/plain_1.tilelist';
    var options = {};
    options.type = 'list';
    options.progress = report;
    options.listStream = fs.createReadStream(list);
    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.end();
    });
});

test('tilelive copy: list auto', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_listauto.mbtiles');
    var options = {};
    options.type = 'list';
    options.progress = report;
    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.end();
    });
});

test('tilelive copy: missing liststream', function(t) {
    var src = 'tilejson+http://api.mapbox.com/v3/mapbox.world-bright.json';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_liststream.mbtiles');
    var options = {};
    options.type = 'list';
    options.progress = report;
    tilelive.copy(src, dst, options, function(err){
        t.ok(err);
        t.equal(err.message, 'You must provide a listStream');
        t.end();
    });
});
<commit_msg>Assert number of tiles copied.<commit_after>var test = require('tape');
var fs = require('fs');
var stream = require('stream');
var tmp = require('os').tmpdir();
var path = require('path');
var exec = require('child_process').exec;
var tilelive = require('../');
var util = require('util');
var concat = require('concat-stream');
var MBTiles = require('mbtiles');
//register protocols
MBTiles.registerProtocols(tilelive);
var crypto = require('crypto');
var Timedsource = require('./timedsource');
var out = [];

var s3url = 's3://tilestream-tilesets-development/carol-staging/mapbox-tile-copy/{z}/{x}/{y}.png';

test('copy usage', function(t) {
    exec(__dirname + '/../bin/tilelive-copy', function(err, stdout, stderr) {
        t.equal(1, err.code, 'exit 1');
        t.ok(/Usage:/.test(stdout), 'shows usage');
        t.end();
    });
});

test('copy copies', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('286/') !== -1, '286');
        t.end();
    });
});

// test('copy copies to s3', function(t) {
//     exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles ' + s3url, function(err, stdout, stderr) {
//         t.ifError(err, 'no errors');
//         t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
//         t.end();
//     });
// });

test('copy min/max', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_minmax.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --minzoom=1 --maxzoom=2 ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('21/') !== -1, '21');
        t.end();
    });
});

test('copy bounds', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_bounds.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --bounds=-180,-85,0,0 ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('59') !== -1, '59');
        t.end();
    });
});

test('copy list', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_list.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --scheme=list --list=' + __dirname + '/fixtures/filescheme.flat ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('100.0000%') !== -1, 'pct complete');
        t.ok(stdout.indexOf('77') !== -1, '77');
        t.end();
    });
});

test('copy streams', function(t) {
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles', {maxBuffer:5e6}, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.indexOf('JSONBREAKFASTTIME\n') === 0);
        t.equal(stdout.length, 647001);
        t.end();
    });
});

test('copy part zero', function(t) {
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles --part 0 --parts 10', function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.equal(stderr, '', 'no stderr');
        t.ok(stdout.split('\n').length < 287, 'does not render all tiles');
        t.end();
    });
});

test('copy timeout', function(t) {
    exec(__dirname + '/../bin/tilelive-copy ' + __dirname + '/fixtures/plain_1.mbtiles --timeout=1', function(err, stdout, stderr) {
        t.ok(err);
        t.ok(/Copy operation timed out/.test(stderr));
        t.end();
    });
});

test('copy slow', function(t) {
    var filepath = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.copy_slow.mbtiles');
    exec(__dirname + '/../bin/tilelive-copy --slow=20 ' + __dirname + '/fixtures/plain_1.mbtiles ' + filepath, function(err, stdout, stderr) {
        t.ifError(err, 'no errors');
        t.ok((/\[slow tile\] (get|put) \d+\/\d+\/\d+ \d+ms/).test(stderr), 'logs slow tiles to stderr');
        t.end();
    });
});

test('tilelive.copy', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy.mbtiles');
    var options = {
        progress: report
    };

    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.end();
    });
});

test('tilelive.copy: concurrency', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_concurrency.mbtiles');
    var options = {
        progress: report,
        concurrency: 10
    };

    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.end();
    });
});

test('tilelive.copy: onslow', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_concurrency.mbtiles');
    var log = [];
    var options = {
        slow: 20,
        onslow: function(method, z, x, y, time) {
            t.equal(/^(get|put)$/.test(method), true);
            t.equal(typeof z, 'number');
            t.equal(typeof x, 'number');
            t.equal(typeof y, 'number');
            t.equal(typeof time, 'number');
            log.push({method:method, z:z, x:x, y:y, time:time});
        },
        progress: report,
        concurrency: 10
    };

    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.ifError(err);
        t.equal(log.length > 0, true, 'calls onslow callback');
        tilelive.stream.slowTime = 60e3;
        t.end();
    });
});

test('tilelive copy: list', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_list.mbtiles');
    var list = __dirname + '/fixtures/plain_1.tilelist';
    var options = {};
    var stats = {};
    options.type = 'list';
    options.progress = function(s) { stats = s; };
    options.listStream = fs.createReadStream(list);
    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.equal(stats.ops, 285, '285 ops');
        t.ifError(err);
        t.end();
    });
});

test('tilelive copy: list auto', function(t) {
    var src = __dirname + '/fixtures/plain_1.mbtiles';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_listauto.mbtiles');
    var options = {};
    var stats = {};
    options.type = 'list';
    options.progress = function(s) { stats = s; };
    tilelive.copy(src, dst, options, function(err){
        if (err) throw err;
        t.equal(stats.ops, 285, '285 ops');
        t.ifError(err);
        t.end();
    });
});

test('tilelive copy: missing liststream', function(t) {
    var src = 'tilejson+http://api.mapbox.com/v3/mapbox.world-bright.json';
    var dst = path.join(tmp, crypto.randomBytes(12).toString('hex') + '.tilelivecopy_liststream.mbtiles');
    var options = {};
    options.type = 'list';
    options.progress = report;
    tilelive.copy(src, dst, options, function(err){
        t.ok(err);
        t.equal(err.message, 'You must provide a listStream');
        t.end();
    });
});

{'fraction_non_alphanumeric': 0.20139904610492845, 'fraction_numerical': 0.018950715421303655, 'mean_word_length': 2.8772189349112427, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1155230', 'n_tokens_mistral': 5553, 'n_tokens_neox': 5370, 'n_words': 1056}