<commit_before>// Copyright, 2013-2014, by <NAME>. <<EMAIL>>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

(function () {
    'use strict';

    var define = require('amdefine')(module);

    /**
     * Array of modules this one depends on.
     * @type {Array}
     */
    var deps = [
        'deferred',
        'sqlite3',
        'util'
    ];

    define(deps, function(deferred, sqlite3, util) {
        var QueueBase = require('./queueBase'),
            Helpers = require('../helper');

        /**
         * SQLite backed queue implementation
         * @param opts Optional options
         * @returns {QueueMemory}
         * @constructor
         */
        function QueueSqlite(opts) {
            opts = opts || {};

            this.sqlite = sqlite3.verbose();
            this.db = null;

            return this;
        }

        util.inherits(QueueSqlite, QueueBase);

        QueueSqlite.prototype.getDb = function() {
            return this.initialize();
        };

        QueueSqlite.prototype.initialize = function() {
            var d = deferred();

            if(this.db) {
                return d.resolve(this.db);
            } else {
                var db = new this.sqlite.Database('queue.db');

                var self = this;
                db.serialize(function() {
                    db.run("DROP TABLE IF EXISTS queue;");

                    db.run("CREATE TABLE IF NOT EXISTS queue (guid TEXT PRIMARY KEY, url TEXT, processor TEXT, category TEXT, raw TEXT);", function(err) {
                        if(err) {
                            return d.reject(err);

                        }

                        // db.run("CREATE UNIQUE INDEX IF  NOT EXISTS idx_url_processor ON queue(url, processor);");

                        self.db = db;
                        d.resolve(self.db);
                    });

                });
            }

            return d.promise();
        }

        /**
         * Get count of items in queue
         * @param queue object or name
         */
        QueueSqlite.prototype.count = function(queue) {
            var d = deferred();

            var self = this;
            this.db.serialize(function() {
                self.db.each("SELECT count(*) FROM queue WHERE category = '" + queue + "'", function (err, row) {
                    if (err) {
                        return d.reject(err);
                    }

                    return d.resolve(row['count(*)']);
                });
            });

            return d.promise();
        };<commit_msg>Add some indexes<commit_after>// Copyright, 2013-2014, by <NAME>. <<EMAIL>>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

(function () {
    'use strict';

    var define = require('amdefine')(module);

    /**
     * Array of modules this one depends on.
     * @type {Array}
     */
    var deps = [
        'deferred',
        'sqlite3',
        'util'
    ];

    define(deps, function(deferred, sqlite3, util) {
        var QueueBase = require('./queueBase'),
            Helpers = require('../helper');

        /**
         * SQLite backed queue implementation
         * @param opts Optional options
         * @returns {QueueMemory}
         * @constructor
         */
        function QueueSqlite(opts) {
            opts = opts || {};

            this.sqlite = sqlite3.verbose();
            this.db = null;

            return this;
        }

        util.inherits(QueueSqlite, QueueBase);

        QueueSqlite.prototype.getDb = function() {
            return this.initialize();
        };

        QueueSqlite.prototype.initialize = function() {
            var d = deferred();

            if(this.db) {
                return d.resolve(this.db);
            } else {
                var db = new this.sqlite.Database('queue.db');

                var self = this;
                db.serialize(function() {
                    db.run("DROP TABLE IF EXISTS queue;");

                    db.run("CREATE TABLE IF NOT EXISTS queue (guid TEXT PRIMARY KEY, url TEXT, processor TEXT, category TEXT, raw TEXT);");

                    db.run("CREATE INDEX IF NOT EXISTS idx_category ON queue(category);");

                    db.run("CREATE INDEX IF NOT EXISTS idx_guid ON queue(guid);");

                    db.run("CREATE INDEX IF NOT EXISTS idx_processor ON queue(processor);");

                    db.run("CREATE INDEX IF NOT EXISTS idx_url ON queue(url);", function(err) {
                        if(err) {
                            return d.reject(err);

                        }

                        // db.run("CREATE UNIQUE INDEX IF  NOT EXISTS idx_url_processor ON queue(url, processor);");

                        self.db = db;
                        d.resolve(self.db);
                    })
                });
            }

            return d.promise();
        }

        /**
         * Get count of items in queue
         * @param queue object or name
         */
        QueueSqlite.prototype.count = function(queue) {
            var d = deferred();

            var self = this;
            this.db.serialize(function() {
                self.db.each("SELECT count(*) FROM queue WHERE category = '" + queue + "'", function (err, row) {
                    if (err) {
                        return d.reject(err);
                    }

                    return d.resolve(row['count(*)']);
                });
            });

            return d.promise();
        };
{'fraction_non_alphanumeric': 0.10362213082128167, 'fraction_numerical': 0.002918933262571315, 'mean_word_length': 1.6787491115849325, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1426924', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1814, 'n_words': 676}