<commit_before>'use strict';

const mongodb = require('mongodb');
const async = require('async');

/**
 * @param {Object} filter
 * @returns {Object}
 */
function buildQuery(filter) {
    const or = filter.map((andFilters) => {
        const and = [];
        andFilters.forEach((condition) => {
            if (condition.operator === 'equal') {
                const c = {};
                if (typeof condition.value === 'object') {
                    // TODO: check if it really is an array
                    c[condition.attribute] = { $in: condition.value };
                } else {
                    c[condition.attribute] = condition.value;
                }
                and.push(c);
            }
        });
        return (and.length === 1) ? and[0] : { $and: and };
    });
    return (or.length === 1) ? or[0] : { $or: or };
}

class DataSource {
    /**
     * @param {Api} api
     * @param {Object} options
     */
    constructor(api, options) {
        this.log = api.log.child({ component: 'flora-mongodb' });
        this.connections = {};

        this.ssl = false;
        this.replicaSet = null;
        this.authSource = null;
        this.baseUrl = 'mongodb://';

        if (options.replicaSet) {
            // Set up Replica Set
            if (!options.servers || !Array.isArray(options.servers)) {
                throw new Error('Invalid configuration, need "servers" object if "replicaSet" is set');
            }

            const hasServerAuth = options.servers.reduce(
                (acc, cur) => !!(cur.username && cur.password) || acc, false);

            if (!hasServerAuth && options.username && options.password) {
                this.baseUrl += options.username + ':' + options.password + '@';
            }

            this.baseUrl += options.servers.map((server) => {
                if (!server.host) throw new Error('Invalid configuration, missing `host` in servers');
                let serverUrl = '';
                if (server.username && server.password) {
                    serverUrl += server.username + ':' + server.password + '@';
                } else if (hasServerAuth && options.username && options.password) {
                    serverUrl += options.username + ':' + options.password + '@';
                }
                serverUrl += server.host + ':' + (server.port || 27017);
                return serverUrl;
            }).join(',');
            this.baseUrl += '/';
            this.ssl = !!options.ssl;
            this.replicaSet = options.replicaSet;
            this.authSource = options.authSource;
        } else {
            // Set up single server
            if (!options.server) throw new Error('Invalid configuration, missing `server`');
            if (!options.server.host) throw new Error('Invalid configuration, missing `host` property in server');
            if (options.username && options.password) this.baseUrl += options.username + ':' + options.password + '@';
            this.baseUrl += options.server.host + ':' + (options.server.port || 27017) + '/';
        }
    }

    prepare() {}

    /**
     * @param {Object} request
     * @param {Function} callback
     */
    process(request, callback) {
        const query = (request.filter) ? buildQuery(request.filter) : {};
        const options = {};

        if (request.attributes) {
            options.fields = {};
            request.attributes.forEach((a) => {
                options.fields[a] = 1;
            });
        }

        if (request.limit) {
            options.limit = request.limit;
        }

        if (request.order) {
            options.sort = request.order.map((o) => {
                const attribute = o.attribute;
                const direction = (o.direction === 'desc' ? -1 : 1);
                return [attribute, direction];
            });
        }

        async.waterfall([
            (next) => {
                this.log.trace('Fetching connection for "%s"', request.database);
                this.getConnection(request.database, next);
            },

            (db, next) => {
                this.log.trace('Fetching collection "%s"', request.collection);
                db.collection(request.collection, next);<commit_msg>added simple skip option if page attribute is set<commit_after>'use strict';

const mongodb = require('mongodb');
const async = require('async');

/**
 * @param {Object} filter
 * @returns {Object}
 */
function buildQuery(filter) {
    const or = filter.map((andFilters) => {
        const and = [];
        andFilters.forEach((condition) => {
            if (condition.operator === 'equal') {
                const c = {};
                if (typeof condition.value === 'object') {
                    // TODO: check if it really is an array
                    c[condition.attribute] = { $in: condition.value };
                } else {
                    c[condition.attribute] = condition.value;
                }
                and.push(c);
            }
        });
        return (and.length === 1) ? and[0] : { $and: and };
    });
    return (or.length === 1) ? or[0] : { $or: or };
}

class DataSource {
    /**
     * @param {Api} api
     * @param {Object} options
     */
    constructor(api, options) {
        this.log = api.log.child({ component: 'flora-mongodb' });
        this.connections = {};

        this.ssl = false;
        this.replicaSet = null;
        this.authSource = null;
        this.baseUrl = 'mongodb://';

        if (options.replicaSet) {
            // Set up Replica Set
            if (!options.servers || !Array.isArray(options.servers)) {
                throw new Error('Invalid configuration, need "servers" object if "replicaSet" is set');
            }

            const hasServerAuth = options.servers.reduce(
                (acc, cur) => !!(cur.username && cur.password) || acc, false);

            if (!hasServerAuth && options.username && options.password) {
                this.baseUrl += options.username + ':' + options.password + '@';
            }

            this.baseUrl += options.servers.map((server) => {
                if (!server.host) throw new Error('Invalid configuration, missing `host` in servers');
                let serverUrl = '';
                if (server.username && server.password) {
                    serverUrl += server.username + ':' + server.password + '@';
                } else if (hasServerAuth && options.username && options.password) {
                    serverUrl += options.username + ':' + options.password + '@';
                }
                serverUrl += server.host + ':' + (server.port || 27017);
                return serverUrl;
            }).join(',');
            this.baseUrl += '/';
            this.ssl = !!options.ssl;
            this.replicaSet = options.replicaSet;
            this.authSource = options.authSource;
        } else {
            // Set up single server
            if (!options.server) throw new Error('Invalid configuration, missing `server`');
            if (!options.server.host) throw new Error('Invalid configuration, missing `host` property in server');
            if (options.username && options.password) this.baseUrl += options.username + ':' + options.password + '@';
            this.baseUrl += options.server.host + ':' + (options.server.port || 27017) + '/';
        }
    }

    prepare() {}

    /**
     * @param {Object} request
     * @param {Function} callback
     */
    process(request, callback) {
        const query = (request.filter) ? buildQuery(request.filter) : {};
        const options = {};

        if (request.attributes) {
            options.fields = {};
            request.attributes.forEach((a) => {
                options.fields[a] = 1;
            });
        }

        if (request.limit) {
            options.limit = request.limit;
        }

        if (request.limit && request.page && request.page > 1) {
            options.skip = request.limit * (request.page - 1);
        }

        if (request.order) {
            options.sort = request.order.map((o) => {
                const attribute = o.attribute;
                const direction = (o.direction === 'desc' ? -1 : 1);
                return [attribute, direction];
            });
        }

        async.waterfall([
            (next) => {
                this.log.trace('Fetching connection for "%s"', request.database);
                this.getConnection(request.database, next);
            },

            (db, next) => {
                this.log.trace('Fetching collection "%s"', request.collection);
                db.collection(request.collection, next);
{'fraction_non_alphanumeric': 0.14625968320036997, 'fraction_numerical': 0.004162330905306972, 'mean_word_length': 1.6648182378311769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1618664', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2249, 'n_words': 516}