<commit_before>/*
   Fathom - Browser-based Network Measurement Platform

   Copyright (C) 2011-2016 Inria Paris-Roquencourt 
                           International Computer Science Institute (ICSI)

   See LICENSE for license and terms of usage. 
*/

/**
 * @fileoverfiew Sqlite DB handler.
 *
 * Fathom uses a sqlite DB for storing a local history of the baseline
 * measurements for the monitoring graphs and other info such as environments.
 *
 * @author <NAME> <<EMAIL>> 
 */
const {Cc, Ci, Cu} = require("chrome");
const { Unknown } = require('sdk/platform/xpcom');
const { all, defer, promised } = require('sdk/core/promise');
const fileIO = require('sdk/io/file');
const ss = require("sdk/simple-storage");
const self = require("sdk/self");
const timers = require("sdk/timers");

Cu.import("resource://gre/modules/Task.jsm"); // exports Task
Cu.import("resource://gre/modules/Sqlite.jsm"); // exports Sqlite
Cu.import("resource://gre/modules/FileUtils.jsm"); // exports FileUtils

const _ = require('underscore');

const {error, FathomException} = require("./error");
const config = require('./config');
const utils = require('./utils');

// current DB schema version
const SCHEMA_VERSION = 9;

/** Singleton accessor. */
var __instance = undefined;
var getInstance = exports.getInstance = function() {
    if (!__instance)
        __instance = new DB();
    return __instance;
};

/** Hidden constructor */
var DB = function() {
    this.version = ss.storage['baseline_dbschema'] || -1;

    this.dbfile = FileUtils.getFile(
        "ProfD", 
        ["fathom", "baseline.sqlite"], 
        true);

    this.dbconf = { 
        path: this.dbfile.path, 
        sharedMemoryCache: false 
    };

    console.log("baselinedb",this.dbfile.path, this.version);

    this.conn = undefined;
};

/** Connect to the db */
DB.prototype.connect = function(cb) {        
    if (this.conn || this.connecting) {
        if (cb) cb();
        return;
    }
    this.connecting = true;

    var that = this;
    Task.spawn(function() {
        that.conn = yield Sqlite.openConnection(that.dbconf);
        that.connecting = false; 
        console.log("baselinedb connected");

        let initdone = yield that.conn.tableExists('baseline');
        if (!initdone) {
            yield that.conn.execute(SQL_CREATE_ENV);
            yield that.conn.execute(SQL_CREATE_BASELINE);
            yield that.conn.execute(SQL_CREATE_AGG1);
            yield that.conn.execute(SQL_CREATE_AGG2);
            yield that.conn.execute(SQL_CREATE_AGG3);    

            ss.storage['baseline_idx'] = -1;
            ss.storage['baseline_idx_agg1'] = -1;
            ss.storage['baseline_idx_agg2'] = -1;
            ss.storage['baseline_idx_agg3'] = -1;

            ss.storage['baseline_dbschema'] = SCHEMA_VERSION;
            that.version = SCHEMA_VERSION;

            console.log('baselinedb tables created schema='+
                ss.storage['baseline_dbschema']);

        } else {
            // do various updates to the current schema depending on
            // the DB version
            console.log('check for DB updates from ' + that.version + ' to ' + SCHEMA_VERSION);

            // updates 5&6&7 & 8 (on win only) require resetting the database (bad values)
            if (that.version < 7 || (that.version < 8 && utils.isWin())) {
                // wipe out and re-create the whole database
                that.conn.close();
                if (that.dbfile && fileIO.exists(that.dbfile.path)) {
                    console.log("baselinedb remove",that.dbfile.path);
                    fileIO.remove(that.dbfile.path);
                }
                console.log("baselinedb re-create",that.dbfile.path);
                that.dbfile = FileUtils.getFile(
                        "ProfD", 
                        ["fathom", "baseline.sqlite"], 
                        true);
                that.dbconf = { 
                    path: that.dbfile.path, 
                    sharedMemoryCache: false 
                };

                // re-connect
                that.conn = yield Sqlite.openConnection(that.dbconf);
                console.log("baselinedb re-connected");

                yield that.conn.execute(SQL_CREATE_ENV);
                yield that.conn.execute(SQL_CREATE_BASELINE);
                yield that.conn.execute(SQL_CREATE_AGG1);
                yield that.conn.execute(SQL_CREATE_AGG2);
                yield that.conn.execute(SQL_CREATE_AGG3);    

                ss.storage['baseline_idx'] = -1;
                ss.storage['baseline_idx_agg1'] = -1;
                ss.storage['baseline_idx_agg2'] = -1;
                ss.storage['baseline_idx_agg3'] = -1;

                console.log('baselinedb tables re-created schema='+SCHEMA_VERSION);

            } else if (that.version < 9) {
                // mlab server info update
                yield that.conn.execute("ALTER TABLE env ADD COL mserver_ip text;")
                yield that.conn.execute("ALTER TABLE env ADD COL mserver_city text;")
                yield that.conn.execute("ALTER TABLE env ADD COL mserver_country text;")
                console.log('baselinedb tables updated to schema='+SCHEMA_VERSION);
            }

            // update version tag
            if (that.version < SCHEMA_VERSION) {
                ss.storage['baseline_dbschema'] = SCHEMA_VERSION;
                that.version = SCHEMA_VERSION;
            }
        }

    }).then(
        cb, // success
        function(err) { // error handler
            that.close();
            console.error('baselinedb',err);
            cb(error("dbconnfailed",err));
        }
    ); // Task
};
<commit_msg>db schema update fix<commit_after>/*
   Fathom - Browser-based Network Measurement Platform

   Copyright (C) 2011-2016 <NAME> 
                           International Computer Science Institute (ICSI)

   See LICENSE for license and terms of usage. 
*/

/**
 * @fileoverfiew Sqlite DB handler.
 *
 * Fathom uses a sqlite DB for storing a local history of the baseline
 * measurements for the monitoring graphs and other info such as environments.
 *
 * @author <NAME> <<EMAIL>> 
 */
const {Cc, Ci, Cu} = require("chrome");
const { Unknown } = require('sdk/platform/xpcom');
const { all, defer, promised } = require('sdk/core/promise');
const fileIO = require('sdk/io/file');
const ss = require("sdk/simple-storage");
const self = require("sdk/self");
const timers = require("sdk/timers");

Cu.import("resource://gre/modules/Task.jsm"); // exports Task
Cu.import("resource://gre/modules/Sqlite.jsm"); // exports Sqlite
Cu.import("resource://gre/modules/FileUtils.jsm"); // exports FileUtils

const _ = require('underscore');

const {error, FathomException} = require("./error");
const config = require('./config');
const utils = require('./utils');

// current DB schema version
const SCHEMA_VERSION = 9;

/** Singleton accessor. */
var __instance = undefined;
var getInstance = exports.getInstance = function() {
    if (!__instance)
        __instance = new DB();
    return __instance;
};

/** Hidden constructor */
var DB = function() {
    this.version = ss.storage['baseline_dbschema'] || -1;

    this.dbfile = FileUtils.getFile(
        "ProfD", 
        ["fathom", "baseline.sqlite"], 
        true);

    this.dbconf = { 
        path: this.dbfile.path, 
        sharedMemoryCache: false 
    };

    console.log("baselinedb",this.dbfile.path, this.version);

    this.conn = undefined;
};

/** Connect to the db */
DB.prototype.connect = function(cb) {        
    if (this.conn || this.connecting) {
        if (cb) cb();
        return;
    }
    this.connecting = true;

    var that = this;
    Task.spawn(function() {
        that.conn = yield Sqlite.openConnection(that.dbconf);
        that.connecting = false; 
        console.log("baselinedb connected");

        let initdone = yield that.conn.tableExists('baseline');

        if (!initdone) {
            yield that.conn.execute(SQL_CREATE_ENV);
            yield that.conn.execute(SQL_CREATE_BASELINE);
            yield that.conn.execute(SQL_CREATE_AGG1);
            yield that.conn.execute(SQL_CREATE_AGG2);
            yield that.conn.execute(SQL_CREATE_AGG3);    

            ss.storage['baseline_idx'] = -1;
            ss.storage['baseline_idx_agg1'] = -1;
            ss.storage['baseline_idx_agg2'] = -1;
            ss.storage['baseline_idx_agg3'] = -1;

            ss.storage['baseline_dbschema'] = SCHEMA_VERSION;
            that.version = SCHEMA_VERSION;

            console.log('baselinedb tables created schema='+
                ss.storage['baseline_dbschema']);

        } else {
            // do various updates to the current schema depending on
            // the DB version
            console.log('check for DB updates from ' + that.version + ' to ' + SCHEMA_VERSION);

            // updates 5&6&7 & 8 (on win only) require resetting the database (bad values)
            if (that.version < 7 || (that.version < 8 && utils.isWin())) {
                // wipe out and re-create the whole database
                that.conn.close();
                if (that.dbfile && fileIO.exists(that.dbfile.path)) {
                    console.log("baselinedb remove",that.dbfile.path);
                    fileIO.remove(that.dbfile.path);
                }
                console.log("baselinedb re-create",that.dbfile.path);
                that.dbfile = FileUtils.getFile(
                        "ProfD", 
                        ["fathom", "baseline.sqlite"], 
                        true);
                that.dbconf = { 
                    path: that.dbfile.path, 
                    sharedMemoryCache: false 
                };

                // re-connect
                that.conn = yield Sqlite.openConnection(that.dbconf);
                console.log("baselinedb re-connected");

                yield that.conn.execute(SQL_CREATE_ENV);
                yield that.conn.execute(SQL_CREATE_BASELINE);
                yield that.conn.execute(SQL_CREATE_AGG1);
                yield that.conn.execute(SQL_CREATE_AGG2);
                yield that.conn.execute(SQL_CREATE_AGG3);    

                ss.storage['baseline_idx'] = -1;
                ss.storage['baseline_idx_agg1'] = -1;
                ss.storage['baseline_idx_agg2'] = -1;
                ss.storage['baseline_idx_agg3'] = -1;

                console.log('baselinedb tables re-created schema='+SCHEMA_VERSION);

            } else if (that.version < 9) {
                // mlab server info update
                yield that.conn.execute("ALTER TABLE env ADD COLUMN mserver_ip text;")
                yield that.conn.execute("ALTER TABLE env ADD COLUMN mserver_city text;")
                yield that.conn.execute("ALTER TABLE env ADD COLUMN mserver_country text;")
                console.log('baselinedb tables updated to schema='+SCHEMA_VERSION);
            }

            // update version tag
            if (that.version < SCHEMA_VERSION) {
                ss.storage['baseline_dbschema'] = SCHEMA_VERSION;
                that.version = SCHEMA_VERSION;
            }
        }

    }).then(
        cb, // success
        function(err) { // error handler
            that.close();
            console.error('baselinedb',err);
            cb(error("dbconnfailed",err));
        }
    ); // Task
};

{'fraction_non_alphanumeric': 0.1239647577092511, 'fraction_numerical': 0.006519823788546255, 'mean_word_length': 1.989465367395312, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5024486', 'n_tokens_mistral': 3375, 'n_tokens_neox': 3142, 'n_words': 716}