<commit_before>var mongo = require("mongodb"),
        app = module.parent.exports.app,
        config = module.parent.exports.config,
        util = require("./util");

var winston = require('winston');

var logger = new (winston.Logger)({
    transports: [
        new (winston.transports.File)({ filename: '/root/concepts-json/node_modules/sct-snapshot-rest-api/search.log' })
    ]
});

app.get('/:db/:collection/concepts/:sctid?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            result = util.flavorize(docs[0], "out");
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});
app.get('/:db/:collection/concepts/:sctid/descriptions/:descriptionId?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            if (o == "limit" || o == "skip") {
                options[o] = parseInt(req.query[o]);
            } else {
                options[o] = JSON.parse(req.query[o]);
            }
        }
    }
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            docs[0].descriptions.forEach(function(desc) {
                                if (req.params.descriptionId) {
                                    if (parseInt(req.params.descriptionId) == desc.descriptionId) {
                                        result.push(desc);
                                    }
                                } else {
                                    result.push(desc);
                                }
                            });
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});
app.get('/:db/:collection/concepts/:sctid/relationships?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var form = "all";
    if (req.query["form"]) {
        form = req.query["form"];
    }

    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }

    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            if (form == "all" || form == "inferred") {
                                docs[0].relationships.forEach(function(desc) {
                                    result.push(desc);
                                });
                            }
                            if (form == "all" || form == "stated") {
                                docs[0].statedRelationships.forEach(function(desc) {
                                    result.push(desc);
                                });
                            }
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});
app.get('/:db/:collection/concepts/:sctid/children?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {"relationships": {"$elemMatch": {"target.conceptId": idParam, "active": true, "type.conceptId": 116680003}}};
    if (req.query["form"]) {
        if (req.query["form"] == "inferred") {
            query = {"relationships": {"$elemMatch": {"target.conceptId": idParam, "active": true, "type.conceptId": 116680003}}};
        }
        if (req.query["form"] == "stated") {
            query = {"statedRelationships": {"$elemMatch": {"target.conceptId": idParam, "active": true, "type.conceptId": 116680003}}};
        }

    }

    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }

    options["fields"] = {"defaultTerm": 1, "conceptId": 1, "active": 1};
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            docs.forEach(function(doc) {
                                result.push(util.flavorize(doc, "out"));
                            });
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send(result);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});

app.get('/:db/:collection/concepts/:sctid/parents?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }
    options["fields"] = {"relationships": 1, "statedRelationships": 1};
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        if (docs.length > 0) {
                            var result = [];
                            if (typeof docs[0].relationships != 'undefined') {
                                if (req.query["form"]) {
                                    if (req.query["form"] == "inferred") {
                                        docs[0].relationships.forEach(function(rel) {
                                            if (rel.active == true && rel.type.conceptId == 116680003) {
                                                result.push({conceptId: rel.target.conceptId, defaultTerm: rel.target.defaultTerm});
                                            }
                                        });
                                    } else if (req.query["form"] == "stated") {
                                        docs[0].statedRelationships.forEach(function(rel) {
                                            if (rel.active == true && rel.type.conceptId == 116680003) {
                                                result.push({conceptId: rel.target.conceptId, defaultTerm: rel.target.defaultTerm});
                                            }
                                        });
                                    }
                                } else {
                                    docs[0].relationships.forEach(function(rel) {
                                        if (rel.active == true && rel.type.conceptId == 116680003) {
                                            result.push({conceptId: rel.target.conceptId, defaultTerm: rel.target.defaultTerm});
                                        }
                                    });
                                }
                            }
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});

app.get('/:db/:collection/descriptions/:sctid?', function(req, res) {
    var idParam = null;
    var query = {'descriptions.descriptionId': 0};
    var searchMode = "startsWith";
    var searchTerm = null;
    var lang = "english";
    if (req.params.sctid) {
        idParam = parseInt(req.params.sctid);
        query = {'descriptionId': idParam};
    } else {
        if (req.query["query"]) {
            if (!req.query["searchMode"] || req.query["searchMode"] == "startsWith") {
                searchTerm = req.query["query"];
                var wildcarded = "^" + req.query["query"] + "x*";
                query = {"term": {"$regex": wildcarded}, "active": true, "conceptActive": true};
            } else if (req.query["searchMode"] == "phraseMatch") {
                searchMode = req.query["searchMode"];
                searchTerm = req.query["query"];
                query = {"text": {"search": searchTerm, "limit": 100, filter: {"active": true}}};
            } else if (req.query["searchMode"] == "wordsAnyOrder") {
                searchMode = req.query["searchMode"];
                searchTerm = req.query["query"];
                var words = searchTerm.split(" ");
                //console.log(JSON.stringify(words));
                query = {"$and": [], "active": true, "conceptActive": true};
                words.forEach(function(word) {
                    //console.log(JSON.stringify(word));
                    var expWord = "^" + word.toLowerCase() + "x*";
                    query.$and.push({"words": {"$regex": expWord}});
                });
            } else {
                res.send([]);
            }
            //console.log(JSON.stringify(query));
        }
    }

    if (req.query["lang"]) {
        lang = req.query["lang"];
    }

    if (searchMode == "startsWith") {
        var options = req.params.options || {};
        var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
        for (o in req.query) {
            if (test.indexOf(o) >= 0) {
                options[o] = JSON.parse(req.query[o]);
            }
        }

        options["limit"] = 100;
        var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
        db.open(function(err, db) {
            db.authenticate(config.db.username, config.db.password, function() {
                db.collection(req.params.collection + 'tx', function(err, collection) {
                    collection.find(query, options, function(err, cursor) {
                        cursor.toArray(function(err, docs) {
                            logger.log('info', 'Search for ' + searchTerm + ' result = ' + docs.length, {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang, matches: docs.length});
                            var result = [];
                            if (docs.length > 0) {
                                docs.forEach(function(doc) {
                                    if (req.params.sctid) {
                                        if (idParam == doc.descriptionId) {
                                            result.push(doc);
                                        }
                                    } else if (searchTerm) {
                                        result.push({"term": doc.term, "conceptId": doc.conceptId, "active": doc.active, "fsn": doc.fsn});
                                    }
                                });
                                res.header('Content-Type', 'application/json');
                                res.send(result);
                            } else {
                                res.send([]);
                            }
                            db.close();
                        });
                    });
                });
            });
        });
    } else if (searchMode == "phraseMatch") {
        var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
        db.open(function(err, db) {
            db.authenticate(config.db.username, config.db.password, function() {
                db.command({text: req.params.collection + 'tx', search: searchTerm, language: lang, filter: {"active": true, "conceptActive": true}}, function(e, cb) {
                    if (e) {
                        console.log(e, 'error');
                        logger.log('error', 'Search for ' + searchTerm + ' ERROR', {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang});
                        res.send(501);
                    } else {
                        //console.log("cb: " + JSON.stringify(cb));
                        logger.log('info', 'Search for ' + searchTerm + ' result = ' + cb.results.length, {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang, matches: cb.results.length});
                        var result = [];
                        var matchedDescriptions = cb.results.slice(0);
                        matchedDescriptions.sort(function(a, b) {
                            if (levDist(a.obj.term, searchTerm) < levDist(b.obj.term, searchTerm))
                                return -1;
                            if (levDist(a.obj.term, searchTerm) > levDist(b.obj.term, searchTerm))
                                return 1;
                            return 0;
                        });
                        if (matchedDescriptions.length > 0) {
                            matchedDescriptions.forEach(function(doc) {
                                result.push({"term": doc.obj.term, "conceptId": doc.obj.conceptId, "active": doc.obj.active, "fsn": doc.obj.fsn});
                            });
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                    }
                    db.close();

                });
            });
        });
    } else if (searchMode == "wordsAnyOrder") {
        var options = req.params.options || {};
        var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
        for (o in req.query) {
            if (test.indexOf(o) >= 0) {
                options[o] = JSON.parse(req.query[o]);
            }
        }
        options["limit"] = 10000000;
        var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, w:0}));
        db.open(function(err, db) {
            db.authenticate(config.db.username, config.db.password, function() {
                db.collection(req.params.collection + "tx", function(err, collection) {
                    collection.find(query, options, function(err, cursor) {
                        cursor.toArray(function(err, docs) {
                            logger.log('info', 'Search for ' + searchTerm + ' result = ' + docs.length, {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang, matches: docs.length});
                            var result = [];
                            if (docs.length > 0) {
                                var matchedDescriptions = docs.slice(0);
                                matchedDescriptions.sort(function(a, b) {
                                    if (a.term.length < b.term.length)
                                        return -1;<commit_msg>Setting the default write concern for the database 6<commit_after>var mongo = require("mongodb"),
        app = module.parent.exports.app,
        config = module.parent.exports.config,
        util = require("./util");

var winston = require('winston');

var logger = new (winston.Logger)({
    transports: [
        new (winston.transports.File)({ filename: '/root/concepts-json/node_modules/sct-snapshot-rest-api/search.log' })
    ]
});

app.get('/:db/:collection/concepts/:sctid?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            result = util.flavorize(docs[0], "out");
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});
app.get('/:db/:collection/concepts/:sctid/descriptions/:descriptionId?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            if (o == "limit" || o == "skip") {
                options[o] = parseInt(req.query[o]);
            } else {
                options[o] = JSON.parse(req.query[o]);
            }
        }
    }
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            docs[0].descriptions.forEach(function(desc) {
                                if (req.params.descriptionId) {
                                    if (parseInt(req.params.descriptionId) == desc.descriptionId) {
                                        result.push(desc);
                                    }
                                } else {
                                    result.push(desc);
                                }
                            });
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});
app.get('/:db/:collection/concepts/:sctid/relationships?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var form = "all";
    if (req.query["form"]) {
        form = req.query["form"];
    }

    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }

    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            if (form == "all" || form == "inferred") {
                                docs[0].relationships.forEach(function(desc) {
                                    result.push(desc);
                                });
                            }
                            if (form == "all" || form == "stated") {
                                docs[0].statedRelationships.forEach(function(desc) {
                                    result.push(desc);
                                });
                            }
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});
app.get('/:db/:collection/concepts/:sctid/children?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {"relationships": {"$elemMatch": {"target.conceptId": idParam, "active": true, "type.conceptId": 116680003}}};
    if (req.query["form"]) {
        if (req.query["form"] == "inferred") {
            query = {"relationships": {"$elemMatch": {"target.conceptId": idParam, "active": true, "type.conceptId": 116680003}}};
        }
        if (req.query["form"] == "stated") {
            query = {"statedRelationships": {"$elemMatch": {"target.conceptId": idParam, "active": true, "type.conceptId": 116680003}}};
        }

    }

    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }

    options["fields"] = {"defaultTerm": 1, "conceptId": 1, "active": 1};
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        var result = [];
                        if (docs.length > 0) {
                            docs.forEach(function(doc) {
                                result.push(util.flavorize(doc, "out"));
                            });
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send(result);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});

app.get('/:db/:collection/concepts/:sctid/parents?', function(req, res) {
    var idParam = parseInt(req.params.sctid);
    var query = {'conceptId': idParam};
    var options = req.params.options || {};
    var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
    for (o in req.query) {
        if (test.indexOf(o) >= 0) {
            options[o] = JSON.parse(req.query[o]);
        }
    }
    options["fields"] = {"relationships": 1, "statedRelationships": 1};
    var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
    db.open(function(err, db) {
        db.authenticate(config.db.username, config.db.password, function() {
            db.collection(req.params.collection, function(err, collection) {
                collection.find(query, options, function(err, cursor) {
                    cursor.toArray(function(err, docs) {
                        if (docs.length > 0) {
                            var result = [];
                            if (typeof docs[0].relationships != 'undefined') {
                                if (req.query["form"]) {
                                    if (req.query["form"] == "inferred") {
                                        docs[0].relationships.forEach(function(rel) {
                                            if (rel.active == true && rel.type.conceptId == 116680003) {
                                                result.push({conceptId: rel.target.conceptId, defaultTerm: rel.target.defaultTerm});
                                            }
                                        });
                                    } else if (req.query["form"] == "stated") {
                                        docs[0].statedRelationships.forEach(function(rel) {
                                            if (rel.active == true && rel.type.conceptId == 116680003) {
                                                result.push({conceptId: rel.target.conceptId, defaultTerm: rel.target.defaultTerm});
                                            }
                                        });
                                    }
                                } else {
                                    docs[0].relationships.forEach(function(rel) {
                                        if (rel.active == true && rel.type.conceptId == 116680003) {
                                            result.push({conceptId: rel.target.conceptId, defaultTerm: rel.target.defaultTerm});
                                        }
                                    });
                                }
                            }
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                        db.close();
                    });
                });
            });
        });
    });
});

app.get('/:db/:collection/descriptions/:sctid?', function(req, res) {
    var idParam = null;
    var query = {'descriptions.descriptionId': 0};
    var searchMode = "startsWith";
    var searchTerm = null;
    var lang = "english";
    if (req.params.sctid) {
        idParam = parseInt(req.params.sctid);
        query = {'descriptionId': idParam};
    } else {
        if (req.query["query"]) {
            if (!req.query["searchMode"] || req.query["searchMode"] == "startsWith") {
                searchTerm = req.query["query"];
                var wildcarded = "^" + req.query["query"] + "x*";
                query = {"term": {"$regex": wildcarded}, "active": true, "conceptActive": true};
            } else if (req.query["searchMode"] == "phraseMatch") {
                searchMode = req.query["searchMode"];
                searchTerm = req.query["query"];
                query = {"text": {"search": searchTerm, "limit": 100, filter: {"active": true}}};
            } else if (req.query["searchMode"] == "wordsAnyOrder") {
                searchMode = req.query["searchMode"];
                searchTerm = req.query["query"];
                var words = searchTerm.split(" ");
                //console.log(JSON.stringify(words));
                query = {"$and": [], "active": true, "conceptActive": true};
                words.forEach(function(word) {
                    //console.log(JSON.stringify(word));
                    var expWord = "^" + word.toLowerCase() + "x*";
                    query.$and.push({"words": {"$regex": expWord}});
                });
            } else {
                res.send([]);
            }
            //console.log(JSON.stringify(query));
        }
    }

    if (req.query["lang"]) {
        lang = req.query["lang"];
    }

    if (searchMode == "startsWith") {
        var options = req.params.options || {};
        var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
        for (o in req.query) {
            if (test.indexOf(o) >= 0) {
                options[o] = JSON.parse(req.query[o]);
            }
        }

        options["limit"] = 100;
        var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
        db.open(function(err, db) {
            db.authenticate(config.db.username, config.db.password, function() {
                db.collection(req.params.collection + 'tx', function(err, collection) {
                    collection.find(query, options, function(err, cursor) {
                        cursor.toArray(function(err, docs) {
                            logger.log('info', 'Search for ' + searchTerm + ' result = ' + docs.length, {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang, matches: docs.length});
                            var result = [];
                            if (docs.length > 0) {
                                docs.forEach(function(doc) {
                                    if (req.params.sctid) {
                                        if (idParam == doc.descriptionId) {
                                            result.push(doc);
                                        }
                                    } else if (searchTerm) {
                                        result.push({"term": doc.term, "conceptId": doc.conceptId, "active": doc.active, "fsn": doc.fsn});
                                    }
                                });
                                res.header('Content-Type', 'application/json');
                                res.send(result);
                            } else {
                                res.send([]);
                            }
                            db.close();
                        });
                    });
                });
            });
        });
    } else if (searchMode == "phraseMatch") {
        var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
        db.open(function(err, db) {
            db.authenticate(config.db.username, config.db.password, function() {
                db.command({text: req.params.collection + 'tx', search: searchTerm, language: lang, filter: {"active": true, "conceptActive": true}}, function(e, cb) {
                    if (e) {
                        console.log(e, 'error');
                        logger.log('error', 'Search for ' + searchTerm + ' ERROR', {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang});
                        res.send(501);
                    } else {
                        //console.log("cb: " + JSON.stringify(cb));
                        logger.log('info', 'Search for ' + searchTerm + ' result = ' + cb.results.length, {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang, matches: cb.results.length});
                        var result = [];
                        var matchedDescriptions = cb.results.slice(0);
                        matchedDescriptions.sort(function(a, b) {
                            if (levDist(a.obj.term, searchTerm) < levDist(b.obj.term, searchTerm))
                                return -1;
                            if (levDist(a.obj.term, searchTerm) > levDist(b.obj.term, searchTerm))
                                return 1;
                            return 0;
                        });
                        if (matchedDescriptions.length > 0) {
                            matchedDescriptions.forEach(function(doc) {
                                result.push({"term": doc.obj.term, "conceptId": doc.obj.conceptId, "active": doc.obj.active, "fsn": doc.obj.fsn});
                            });
                            res.header('Content-Type', 'application/json');
                            res.send(result);
                        } else {
                            res.send([]);
                        }
                    }
                    db.close();

                });
            });
        });
    } else if (searchMode == "wordsAnyOrder") {
        var options = req.params.options || {};
        var test = ['limit', 'sort', 'fields', 'skip', 'hint', 'explain', 'snapshot', 'timeout'];
        for (o in req.query) {
            if (test.indexOf(o) >= 0) {
                options[o] = JSON.parse(req.query[o]);
            }
        }
        options["limit"] = 10000000;
        var db = new mongo.Db(req.params.db, new mongo.Server(config.db.host, config.db.port, {'auto_reconnect': true, safe:false}));
        db.open(function(err, db) {
            db.authenticate(config.db.username, config.db.password, function() {
                db.collection(req.params.collection + "tx", function(err, collection) {
                    collection.find(query, options, function(err, cursor) {
                        cursor.toArray(function(err, docs) {
                            logger.log('info', 'Search for ' + searchTerm + ' result = ' + docs.length, {searchTerm: searchTerm, database: req.params.db, collection: req.params.collection, searchMode: searchMode, language: lang, matches: docs.length});
                            var result = [];
                            if (docs.length > 0) {
                                var matchedDescriptions = docs.slice(0);
                                matchedDescriptions.sort(function(a, b) {
                                    if (a.term.length < b.term.length)
                                        return -1;
{'fraction_non_alphanumeric': 0.14570539699715948, 'fraction_numerical': 0.005978628432300825, 'mean_word_length': 1.3166008648242151, 'pattern_counts': {'":': 86, '<': 7, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 100, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709563', 'n_tokens_mistral': 9942, 'n_tokens_neox': 9401, 'n_words': 1779}