<commit_before>/**
 * @module NoSQL Embedded Database
 * @author <NAME> <<EMAIL>>
 * @copyright <NAME> 2012-2015
 * @version 3.0.0
 */

'use strict';

var fs = require('fs');
var path = require('path');
var util = require('util');
var events = require('events');

var VERSION = 'v3.0.0';
var STATUS_UNKNOWN = 0;
var STATUS_READING = 1;
var STATUS_WRITING = 2;
var STATUS_LOCKING = 3;
var STATUS_PENDING = 4;
var EXTENSION = '.nosql';
var EXTENSION_VIEW = '.nosql';
var EXTENSION_BINARY = '.nosql-binary';
var EXTENSION_TMP = '.nosql-tmp';
var EXTENSION_CHANGES = '.changelog';
var EXTENSION_STORED = '.nosql-stored';
var EXTENSION_META = '.meta';
var MAX_WRITESTREAM = 2;
var MAX_READSTREAM = 4;
var MAX_BUFFER_SIZE = 1024 * 40;
var BINARY_HEADER_LENGTH = 2000;
var NEWLINE = '\n';
var STRING = 'string';
var FUNCTION = 'function';
var UNDEFINED = 'undefined';
var BOOLEAN = 'boolean';

if (typeof(setImmediate) === UNDEFINED) {
    global.setImmediate = function(cb) {
        process.nextTick(cb);
    };
}

/*
    Database
    @filename {String}
    @directory {String} :: for storing binary data
    @changes {Boolean} :: optional, default true
*/
function Database(filename, directory, changes) {

    if (typeof(directory) === BOOLEAN) {
        changes = directory;
        directory = null;
    }

    this.isReady = false;
    this.status_prev = STATUS_UNKNOWN;
    this.status = STATUS_UNKNOWN;
    this.changes = changes === undefined ? true : changes === true;

    this.countRead = 0;
    this.countWrite = 0;

    this.pendingRead = [];
    this.pendingEach = [];
    this.pendingLock = [];
    this.pendingDrop = [];
    this.pendingWrite = [];
    this.pendingClear = [];

    this.isPending = false;

    if (filename.indexOf(EXTENSION) !== -1)
        filename = filename.replace(EXTENSION, '');

    this.filename = filename + EXTENSION;
    this.filenameTemp = filename + EXTENSION_TMP;
    this.filenameChanges = filename + EXTENSION_CHANGES;
    this.filenameStored = filename + EXTENSION_STORED;
    this.filenameMeta = filename + EXTENSION_META;

    this.name = path.basename(filename);

    this.directory = path.dirname(filename);
    this.views = new Views(this);

    this.meta = {
        version: VERSION,
        views: {},
        stored: {},
        description: '',
        created: new Date(),
        custom: null
    };

    this.binary = (directory || '').length === 0 ? null : new Binary(this, directory);
    this.changelog = new Changelog(this, this.filenameChanges);
    this.file = new FileReader(this);
    this.stored = new Stored(this, this.filenameStored);

    this._metaLoad();
}

/*
    @db {Database}
*/
function Views(db) {
    this.views = {};
    this.db = db;
    this.directory = db.directory;
    this.emit = db.emit;
}

/*
    @db {Database}
    @name {String}
    @filename {String} :: database filename
*/
function View(db, name, filename) {
    this.db = db;
    this.status = STATUS_UNKNOWN;
    this.countRead = 0;
    this.pendingRead = [];
    this.pendingOperation = [];
    this.filename = filename;
    this.name = name;
    this.emit = db.emit;
    this.file = new FileReader(db);
}

/*
    @db {Database}
*/
function Stored(db, filename) {
    this.filename = filename;
    this.db = db;
    this.stored = {};
    this.cache = {};
    this.isReaded = false;
}

/*
    @db {Database}
    @directory {String}
*/
function Binary(db, directory) {
    this.db = db;
    this.directory = directory;
    this.exists = false;
}

/*
    @db {Database}
    @filename {String}
*/
function Changelog(db, filename) {
    this.filename = filename;
    this.db = db;
}

/*
    @db {Database}
*/
function FileReader(db) {
    this.db = db;
}

/*
    PROTOTYPES
*/

Database.prototype = {

    get created() {

        var dt = this.meta.created;
        if (util.isDate(dt))
            return dt;

        if (dt === null || dt === undefined)
            return null;

        return new Date(Date.parse(dt.toString()));
    }
};

Database.prototype.__proto__ = Object.create(events.EventEmitter.prototype, {
    constructor: {
        value: Database,
        enumberable: false
    }
});

/*
    Insert data into database
    @arr {Array of Object}
    @fnCallback {Function(err, count)} callback
    @changes {String} :: optional, insert description
    return {Database}
*/
Database.prototype.insert = function(arr, fnCallback, changes) {

    var self = this;

    if (typeof(fnCallback) === STRING) {
        changes = fnCallback;
        fnCallback = null;
    }

    if (!(arr instanceof Array))
        arr = [arr];

    var length = arr.length;

    if (self.status === STATUS_LOCKING|| self.status === STATUS_PENDING || self.countWrite >= MAX_WRITESTREAM) {

        for (var i = 0; i < length; i++)
            self.pendingWrite.push({ json: arr[i], changes: changes });

        if (fnCallback)
            fnCallback(null, -1);

        return self;
    }

    var builder = [];
    var builderChanges = [];

    for (var i = 0; i < length; i++) {

        var doc = arr[i];

        if (doc.json === undefined) {
            builder.push(doc);

            if (changes)
                builderChanges.push(changes);

            continue;
        }

        builder.push(doc.json);

        if (doc.changes)
            builderChanges.push(doc.changes);
    }

    if (builder.length === 0) {
        self.next();
        return;
    }

    self.emit('insert', true, builder.length);

    self.status = STATUS_WRITING;
    self.countWrite++;

    appendFile(self.filename, builder, function() {

        self.countWrite--;
        self.emit('insert', false, builder.length);
        self.next();

        self.changelog.insert(builderChanges);

        if (fnCallback) {
            var length = builder.length;
            setImmediate(function() { fnCallback(null, length); });
        }

        builder = null;
        builderChanges = null;
        arr = null;

        var keys = Object.keys(self.meta.views);
        var length = keys.length;
        for (var i = 0; i < length; i++)
            self.views.refresh(keys[i]);

    }, self);

    return self;
};

/*
    Read data from database
    @fnMap {Function} :: params: @doc {Object}, IMPORTANT: you must return {Object}
    @fnCallback {Function(err, array/number)}
    @itemSkip {Number} :: optional, default 0
    @itemTake {Number} :: optional, defualt 0
    @isScalar {Boolean} :: optional, default is false
    return {Database}
*/
Database.prototype.read = function(fnMap, fnCallback, itemSkip, itemTake, isScalar, name) {

    var self = this;
    var skip = itemSkip || 0;
    var take = itemTake || 0;

    if (self.status === STATUS_LOCKING || self.status === STATUS_PENDING || self.countRead >= MAX_READSTREAM) {

        self.pendingRead.push(function() {
            self.read(fnMap, fnCallback, itemSkip, itemTake, isScalar);
        });

        return self;
    }

    if (fnCallback === undefined) {
        fnCallback = fnMap;
        fnMap = function(doc) { return doc; };
    }

    if (typeof(fnMap) === STRING)
        fnMap = filterPrepare(fnMap);

    if (fnMap === null)
        fnMap = function(doc) { return doc; };

    self.emit(name || 'read', true, 0);

    // opened streams
    self.countRead++;
    self.status = STATUS_READING;

    var selected = [];
    var current = '';
    var count = 0;
    var resume = true;

    var fnBuffer = function(buffer) {
        current += buffer;
        return current;
    };

    var fnItem = function(err, doc) {

        if (!resume)
            return;

        // clear buffer;
        current = '';

        if (err)
            return;

        var item = fnMap(doc);
        if (item === false || item === null || item === undefined)
            return;

        count++;

        if (skip > 0 && count <= skip)
            return;

        if (!isScalar)
            selected.push(item === true ? doc : item);

        if (take > 0 && selected.length === take)
            resume = false;
    };

    var reader = self.file.open(self.filename, MAX_BUFFER_SIZE, function(buffer) {

        onBuffer(buffer, fnItem, fnBuffer);
        return resume;

    }, function() {
        self.countRead--;
        self.next();

        setImmediate(function() {
            self.emit(name || 'read', false, isScalar ? count : selected.length);
            fnCallback(null, isScalar ? count : selected);
        });
    });

    return self;
};

/*
    Read all documents from database
    @fnMap {Function} :: IMPORTANT: you must return {Object}
    @fnCallback {Function(err, array)}
    @itemSkip {Number} :: optional, default 0
    @itemTake {Number} :: optional, default 0
    return {Database}
*/
Database.prototype.all = function(fnMap, fnCallback, itemSkip, itemTake) {
    return this.read(fnMap, fnCallback, itemSkip, itemTake, false, 'all');
};

/*
    Read one document from database
    @fnMap {Function} :: must return {Object}
    @fnCallback {Function(err, doc)}
    return {Database}
*/
Database.prototype.one = function(fnMap, fnCallback) {

    var cb = function(selected) {
        fnCallback(null, selected ? selected[0] || null : null);
    };

    return this.read(fnMap, cb, 0, 1, false, 'one');
};

/*
    Read TOP "x" documents from database
    @fnMap {Function} :: IMPORTANT: you must return {Object}
    @fnCallback {Function} :: params: @doc {Array of Object}
    return {Database}
*/
Database.prototype.top = function(max, fnMap, fnCallback) {
    return this.read(fnMap, fnCallback, 0, max, false, 'top');
};

/*
    Count documents
    @fnFilter {Function} :: params: @doc {Object}, IMPORTANT: you must return {Boolean}
    @fnCallback {Function} :: params: @count {Number}
    return {Database}
*/
Database.prototype.count = function(fnFilter, fnCallback) {
    return this.read(fnFilter, fnCallback, 0, 0, true, 'count');
};

/*
    Read each document from database<commit_msg>Fix nosql.one that was not returning the results<commit_after>/**
 * @module NoSQL Embedded Database
 * @author <NAME> <<EMAIL>>
 * @copyright <NAME> 2012-2015
 * @version 3.0.0
 */

'use strict';

var fs = require('fs');
var path = require('path');
var util = require('util');
var events = require('events');

var VERSION = 'v3.0.0';
var STATUS_UNKNOWN = 0;
var STATUS_READING = 1;
var STATUS_WRITING = 2;
var STATUS_LOCKING = 3;
var STATUS_PENDING = 4;
var EXTENSION = '.nosql';
var EXTENSION_VIEW = '.nosql';
var EXTENSION_BINARY = '.nosql-binary';
var EXTENSION_TMP = '.nosql-tmp';
var EXTENSION_CHANGES = '.changelog';
var EXTENSION_STORED = '.nosql-stored';
var EXTENSION_META = '.meta';
var MAX_WRITESTREAM = 2;
var MAX_READSTREAM = 4;
var MAX_BUFFER_SIZE = 1024 * 40;
var BINARY_HEADER_LENGTH = 2000;
var NEWLINE = '\n';
var STRING = 'string';
var FUNCTION = 'function';
var UNDEFINED = 'undefined';
var BOOLEAN = 'boolean';

if (typeof(setImmediate) === UNDEFINED) {
    global.setImmediate = function(cb) {
        process.nextTick(cb);
    };
}

/*
    Database
    @filename {String}
    @directory {String} :: for storing binary data
    @changes {Boolean} :: optional, default true
*/
function Database(filename, directory, changes) {

    if (typeof(directory) === BOOLEAN) {
        changes = directory;
        directory = null;
    }

    this.isReady = false;
    this.status_prev = STATUS_UNKNOWN;
    this.status = STATUS_UNKNOWN;
    this.changes = changes === undefined ? true : changes === true;

    this.countRead = 0;
    this.countWrite = 0;

    this.pendingRead = [];
    this.pendingEach = [];
    this.pendingLock = [];
    this.pendingDrop = [];
    this.pendingWrite = [];
    this.pendingClear = [];

    this.isPending = false;

    if (filename.indexOf(EXTENSION) !== -1)
        filename = filename.replace(EXTENSION, '');

    this.filename = filename + EXTENSION;
    this.filenameTemp = filename + EXTENSION_TMP;
    this.filenameChanges = filename + EXTENSION_CHANGES;
    this.filenameStored = filename + EXTENSION_STORED;
    this.filenameMeta = filename + EXTENSION_META;

    this.name = path.basename(filename);

    this.directory = path.dirname(filename);
    this.views = new Views(this);

    this.meta = {
        version: VERSION,
        views: {},
        stored: {},
        description: '',
        created: new Date(),
        custom: null
    };

    this.binary = (directory || '').length === 0 ? null : new Binary(this, directory);
    this.changelog = new Changelog(this, this.filenameChanges);
    this.file = new FileReader(this);
    this.stored = new Stored(this, this.filenameStored);

    this._metaLoad();
}

/*
    @db {Database}
*/
function Views(db) {
    this.views = {};
    this.db = db;
    this.directory = db.directory;
    this.emit = db.emit;
}

/*
    @db {Database}
    @name {String}
    @filename {String} :: database filename
*/
function View(db, name, filename) {
    this.db = db;
    this.status = STATUS_UNKNOWN;
    this.countRead = 0;
    this.pendingRead = [];
    this.pendingOperation = [];
    this.filename = filename;
    this.name = name;
    this.emit = db.emit;
    this.file = new FileReader(db);
}

/*
    @db {Database}
*/
function Stored(db, filename) {
    this.filename = filename;
    this.db = db;
    this.stored = {};
    this.cache = {};
    this.isReaded = false;
}

/*
    @db {Database}
    @directory {String}
*/
function Binary(db, directory) {
    this.db = db;
    this.directory = directory;
    this.exists = false;
}

/*
    @db {Database}
    @filename {String}
*/
function Changelog(db, filename) {
    this.filename = filename;
    this.db = db;
}

/*
    @db {Database}
*/
function FileReader(db) {
    this.db = db;
}

/*
    PROTOTYPES
*/

Database.prototype = {

    get created() {

        var dt = this.meta.created;
        if (util.isDate(dt))
            return dt;

        if (dt === null || dt === undefined)
            return null;

        return new Date(Date.parse(dt.toString()));
    }
};

Database.prototype.__proto__ = Object.create(events.EventEmitter.prototype, {
    constructor: {
        value: Database,
        enumberable: false
    }
});

/*
    Insert data into database
    @arr {Array of Object}
    @fnCallback {Function(err, count)} callback
    @changes {String} :: optional, insert description
    return {Database}
*/
Database.prototype.insert = function(arr, fnCallback, changes) {

    var self = this;

    if (typeof(fnCallback) === STRING) {
        changes = fnCallback;
        fnCallback = null;
    }

    if (!(arr instanceof Array))
        arr = [arr];

    var length = arr.length;

    if (self.status === STATUS_LOCKING|| self.status === STATUS_PENDING || self.countWrite >= MAX_WRITESTREAM) {

        for (var i = 0; i < length; i++)
            self.pendingWrite.push({ json: arr[i], changes: changes });

        if (fnCallback)
            fnCallback(null, -1);

        return self;
    }

    var builder = [];
    var builderChanges = [];

    for (var i = 0; i < length; i++) {

        var doc = arr[i];

        if (doc.json === undefined) {
            builder.push(doc);

            if (changes)
                builderChanges.push(changes);

            continue;
        }

        builder.push(doc.json);

        if (doc.changes)
            builderChanges.push(doc.changes);
    }

    if (builder.length === 0) {
        self.next();
        return;
    }

    self.emit('insert', true, builder.length);

    self.status = STATUS_WRITING;
    self.countWrite++;

    appendFile(self.filename, builder, function() {

        self.countWrite--;
        self.emit('insert', false, builder.length);
        self.next();

        self.changelog.insert(builderChanges);

        if (fnCallback) {
            var length = builder.length;
            setImmediate(function() { fnCallback(null, length); });
        }

        builder = null;
        builderChanges = null;
        arr = null;

        var keys = Object.keys(self.meta.views);
        var length = keys.length;
        for (var i = 0; i < length; i++)
            self.views.refresh(keys[i]);

    }, self);

    return self;
};

/*
    Read data from database
    @fnMap {Function} :: params: @doc {Object}, IMPORTANT: you must return {Object}
    @fnCallback {Function(err, array/number)}
    @itemSkip {Number} :: optional, default 0
    @itemTake {Number} :: optional, defualt 0
    @isScalar {Boolean} :: optional, default is false
    return {Database}
*/
Database.prototype.read = function(fnMap, fnCallback, itemSkip, itemTake, isScalar, name) {

    var self = this;
    var skip = itemSkip || 0;
    var take = itemTake || 0;

    if (self.status === STATUS_LOCKING || self.status === STATUS_PENDING || self.countRead >= MAX_READSTREAM) {

        self.pendingRead.push(function() {
            self.read(fnMap, fnCallback, itemSkip, itemTake, isScalar);
        });

        return self;
    }

    if (fnCallback === undefined) {
        fnCallback = fnMap;
        fnMap = function(doc) { return doc; };
    }

    if (typeof(fnMap) === STRING)
        fnMap = filterPrepare(fnMap);

    if (fnMap === null)
        fnMap = function(doc) { return doc; };

    self.emit(name || 'read', true, 0);

    // opened streams
    self.countRead++;
    self.status = STATUS_READING;

    var selected = [];
    var current = '';
    var count = 0;
    var resume = true;

    var fnBuffer = function(buffer) {
        current += buffer;
        return current;
    };

    var fnItem = function(err, doc) {

        if (!resume)
            return;

        // clear buffer;
        current = '';

        if (err)
            return;

        var item = fnMap(doc);
        if (item === false || item === null || item === undefined)
            return;

        count++;

        if (skip > 0 && count <= skip)
            return;

        if (!isScalar)
            selected.push(item === true ? doc : item);

        if (take > 0 && selected.length === take)
            resume = false;
    };

    var reader = self.file.open(self.filename, MAX_BUFFER_SIZE, function(buffer) {

        onBuffer(buffer, fnItem, fnBuffer);
        return resume;

    }, function() {
        self.countRead--;
        self.next();

        setImmediate(function() {
            self.emit(name || 'read', false, isScalar ? count : selected.length);
            fnCallback(null, isScalar ? count : selected);
        });
    });

    return self;
};

/*
    Read all documents from database
    @fnMap {Function} :: IMPORTANT: you must return {Object}
    @fnCallback {Function(err, array)}
    @itemSkip {Number} :: optional, default 0
    @itemTake {Number} :: optional, default 0
    return {Database}
*/
Database.prototype.all = function(fnMap, fnCallback, itemSkip, itemTake) {
    return this.read(fnMap, fnCallback, itemSkip, itemTake, false, 'all');
};

/*
    Read one document from database
    @fnMap {Function} :: must return {Object}
    @fnCallback {Function(err, doc)}
    return {Database}
*/
Database.prototype.one = function(fnMap, fnCallback) {

    var cb = function(err, selected) {
        fnCallback(err, selected ? selected[0] || null : null);
    };

    return this.read(fnMap, cb, 0, 1, false, 'one');
};

/*
    Read TOP "x" documents from database
    @fnMap {Function} :: IMPORTANT: you must return {Object}
    @fnCallback {Function} :: params: @doc {Array of Object}
    return {Database}
*/
Database.prototype.top = function(max, fnMap, fnCallback) {
    return this.read(fnMap, fnCallback, 0, max, false, 'top');
};

/*
    Count documents
    @fnFilter {Function} :: params: @doc {Object}, IMPORTANT: you must return {Boolean}
    @fnCallback {Function} :: params: @count {Number}
    return {Database}
*/
Database.prototype.count = function(fnFilter, fnCallback) {
    return this.read(fnFilter, fnCallback, 0, 0, true, 'count');
};

/*
    Read each document from database
{'fraction_non_alphanumeric': 0.14100951134819587, 'fraction_numerical': 0.005737003673695335, 'mean_word_length': 2.4875394875394874, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1280988', 'n_tokens_mistral': 6355, 'n_tokens_neox': 6138, 'n_words': 1579}