<commit_before>var _ = require('underscore');
var fs = require('fs');
var url = require('url');
var path = require('path');
var yaml = require('js-yaml');
var mapnik = require('mapnik');
var mkdirp = require('mkdirp');
var chrono = require('chrono');
var crypto = require('crypto');
var tilelive = require('tilelive');

tilelive.protocols['mbtiles:'] = require('mbtiles');

// Register default fonts.
mapnik.register_default_fonts();
mapnik.register_fonts(path.dirname(require.resolve('tm2-default-fonts')), { recurse: true });

var tm = {};

// Set or get config.
tm.db;
tm._config = {};
tm.config = function(opts, callback) {
    if (!opts) return tm._config;

    tm._config = _(opts).defaults(tm._config);
    // @TODO create other paths (cache, etc.)
    try {
        mkdirp.sync(path.dirname(tm.config().db));
        mkdirp.sync(tm.config().cache);
    } catch(err) { throw err; }

    // Load + compact the app database.
    var old = require('dirty')(tm.config().db);
    old.on('load', function() {
        fs.unlinkSync(tm.config().db);
        tm.db = require('dirty')(tm.config().db);

        // Check app db version and run any migrations if necessary.
        switch (old.get('version')) {
        case 2:
            old.forEach(function(k,v) { tm.db.set(k,v); });
            break;
        case 1:
        case undefined:
            old.set('version', 2);
            old.set('history', _(old.get('history')||{}).reduce(function(memo, list, type) {
                if (type !== 'style' && type !== 'source') return memo;
                memo[type] = list.map(function(id) {
                    var uri = url.parse(id);
                    if (uri.protocol === 'mapbox:') {<commit_msg>only clear out app.db if it exists<commit_after>var _ = require('underscore');
var fs = require('fs');
var url = require('url');
var path = require('path');
var yaml = require('js-yaml');
var mapnik = require('mapnik');
var mkdirp = require('mkdirp');
var chrono = require('chrono');
var crypto = require('crypto');
var tilelive = require('tilelive');
var existsSync = require('fs').existsSync || require('path').existsSync;

tilelive.protocols['mbtiles:'] = require('mbtiles');

// Register default fonts.
mapnik.register_default_fonts();
mapnik.register_fonts(path.dirname(require.resolve('tm2-default-fonts')), { recurse: true });

var tm = {};

// Set or get config.
tm.db;
tm._config = {};
tm.config = function(opts, callback) {
    if (!opts) return tm._config;

    tm._config = _(opts).defaults(tm._config);
    // @TODO create other paths (cache, etc.)
    try {
        mkdirp.sync(path.dirname(tm.config().db));
        mkdirp.sync(tm.config().cache);
    } catch(err) { throw err; }

    // Load + compact the app database.
    var old = require('dirty')(tm.config().db);
    old.on('load', function() {
        var db_name = tm.config().db;
        if (existsSync(db_name)) {
            fs.unlinkSync(db_name);
        }
        tm.db = require('dirty')(db_name);

        // Check app db version and run any migrations if necessary.
        switch (old.get('version')) {
        case 2:
            old.forEach(function(k,v) { tm.db.set(k,v); });
            break;
        case 1:
        case undefined:
            old.set('version', 2);
            old.set('history', _(old.get('history')||{}).reduce(function(memo, list, type) {
                if (type !== 'style' && type !== 'source') return memo;
                memo[type] = list.map(function(id) {
                    var uri = url.parse(id);
                    if (uri.protocol === 'mapbox:') {
{'fraction_non_alphanumeric': 0.17920044730220855, 'fraction_numerical': 0.0022365110427732737, 'mean_word_length': 2.7270833333333333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1288571', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1110, 'n_words': 259}