<commit_before>/*
*
* Copyright (C) 2011, The Locker Project
* All rights reserved.
*
* Please see the LICENSE file for more information.
*
*/

var fs = require('fs'),
    locker = require('../../Common/node/locker.js');

var lsearch = require('../../Common/node/lsearch');
var lutil = require('../../Common/node/lutil');
var lconfig = require('lconfig');
lconfig.load('Config/config.json');

var lockerInfo = {};
exports.lockerInfo = lockerInfo;

var express = require('express'),
    connect = require('connect');
var request = require('request');
var async = require('async');
var app = express.createServer(connect.bodyParser());

var maxCloseTimeout = null;
var MAX_CLOSE_TIMEOUT = 10000;

app.set('views', __dirname);

app.get('/', function(req, res) {
    res.send("You should use a search interface instead of trying to talk to me directly.");
});

app.post('/events', function(req, res) {
    exports.handlePostEvents(req, function(err, response) {
        if (err) {
            return res.send(err, 500);
        }
        return res.send(response);
    });
});

app.post('/index', function(req, res) {
    exports.handlePostIndex(req, function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send(response);
   });
});

app.get('/query', function(req, res) {
    exports.handleGetQuery(req, function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send(response);
   });
});

app.get('/update', function(req, res) {
    exports.handleGetUpdate(function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send('Full search reindex started');
   });
});

app.get('/reindexForType', function(req, res) {
    exports.handleGetReindexForType(req.param('type'), function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send(response);
   });
});

exports.handleGetUpdate = function(callback) {
    var error;
    lsearch.resetIndex(function(err) {
        if (err) {
            error = 'Failed attempting to reset search index for /search/update GET request: ' + err;
            console.error(error);
            return callback(err);
        }

        reindexType(lockerInfo.lockerUrl + '/Me/contacts/allContacts', 'contact/full', 'contacts', function(err) {});
        reindexType(lockerInfo.lockerUrl + '/Me/photos/allPhotos', 'photo/full', 'photos', function(err) {});
        locker.providers('status/twitter', function(err, services) {
            if (!services) return;
            services.forEach(function(svc) {
               if (svc.provides.indexOf('status/twitter') >= 0) {
                   reindexType(lockerInfo.lockerUrl + '/Me/' + svc.id + '/getCurrent/timeline', 'timeline/twitter', 'twitter/timeline', function(err) {});
                }
            });
        });

        return callback(err);
    });
};

exports.handlePostEvents = function(req, callback) {
    var error;

    if (req.headers['content-type'] !== 'application/json') {
        error = 'Expected content-type of "application/json" for /search/events POST request. Received content-type: ' + req.headers['content-type'];
        console.error(error);
        return callback(error, {});
    }

    if (!req.body) {
        error = 'Empty body received for /search/events POST request.';
        console.error(error);
        return callback(error, {});
    }

    if (req.body.hasOwnProperty('type')) {
        // FIXME Hack to handle inconsistencies between photo and contacts collection
        if (req.body.type === 'photo') {
            req.body.type = 'photo/full';
        }
        // END FIXME

        var source = getSourceForEvent(req.body);

        if (req.body.action === 'new' || req.body.action === 'update') {
            lsearch.indexTypeAndSource(req.body.type, source, req.body.obj.data, function(err, time) {
                if (err) {
                    handleError(req.body.type, req.body.action, req.body.obj.data._id, err);
                    return callback(err, {});
                }
                handleLog(req.body.type, req.body.action, req.body.obj.data._id, time);
                if (maxCloseTimeout) clearTimeout(maxCloseTimeout);
                maxCloseTimeout = setTimeout(function() {
                    lsearch.flushAndCloseWriter();
                }, MAX_CLOSE_TIMEOUT);
                return callback(err, {timeToIndex: time});
            });
        } else if (req.body.action === 'delete') {
            lsearch.deleteDocument(req.body.obj.data._id, function(err, time) {
                if (err) {
                    handleError(req.body.type, req.body.action, req.body.obj.data._id, err);
                    return callback(err, {});
                }
                handleLog(req.body.type, req.body.action, req.body.obj.data._id, time);
                return callback(err, {timeToIndex: time});
            });
        } else {
            console.log("Unexpected event: " + req.body.type + " and " + req.body.action);
            res.end();
        }
    } else {
        console.log("Unexpected event or not json " + req.headers["content-type"]);
        res.end();
    }
};

exports.handlePostIndex = function(req, callback) {
    var error;

    if (!req.body.type || !req.body.source || !req.body.data) {
        error = 'Invalid arguments given for /search/index POST request.';<commit_msg>add more debug to track down unusual undefined<commit_after>/*
*
* Copyright (C) 2011, The Locker Project
* All rights reserved.
*
* Please see the LICENSE file for more information.
*
*/

var fs = require('fs'),
    locker = require('../../Common/node/locker.js');

var lsearch = require('../../Common/node/lsearch');
var lutil = require('../../Common/node/lutil');
var lconfig = require('lconfig');
lconfig.load('Config/config.json');

var lockerInfo = {};
exports.lockerInfo = lockerInfo;

var express = require('express'),
    connect = require('connect');
var request = require('request');
var async = require('async');
var app = express.createServer(connect.bodyParser());

var maxCloseTimeout = null;
var MAX_CLOSE_TIMEOUT = 10000;

app.set('views', __dirname);

app.get('/', function(req, res) {
    res.send("You should use a search interface instead of trying to talk to me directly.");
});

app.post('/events', function(req, res) {
    exports.handlePostEvents(req, function(err, response) {
        if (err) {
            return res.send(err, 500);
        }
        return res.send(response);
    });
});

app.post('/index', function(req, res) {
    exports.handlePostIndex(req, function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send(response);
   });
});

app.get('/query', function(req, res) {
    exports.handleGetQuery(req, function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send(response);
   });
});

app.get('/update', function(req, res) {
    exports.handleGetUpdate(function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send('Full search reindex started');
   });
});

app.get('/reindexForType', function(req, res) {
    exports.handleGetReindexForType(req.param('type'), function(err, response) {
       if (err) {
           return res.send(err, 500);
       }
       return res.send(response);
   });
});

exports.handleGetUpdate = function(callback) {
    var error;
    lsearch.resetIndex(function(err) {
        if (err) {
            error = 'Failed attempting to reset search index for /search/update GET request: ' + err;
            console.error(error);
            return callback(err);
        }

        reindexType(lockerInfo.lockerUrl + '/Me/contacts/allContacts', 'contact/full', 'contacts', function(err) {});
        reindexType(lockerInfo.lockerUrl + '/Me/photos/allPhotos', 'photo/full', 'photos', function(err) {});
        locker.providers('status/twitter', function(err, services) {
            if (!services) return;
            services.forEach(function(svc) {
               if (svc.provides.indexOf('status/twitter') >= 0) {
                   reindexType(lockerInfo.lockerUrl + '/Me/' + svc.id + '/getCurrent/timeline', 'timeline/twitter', 'twitter/timeline', function(err) {});
                }
            });
        });

        return callback(err);
    });
};

exports.handlePostEvents = function(req, callback) {
    var error;

    if (req.headers['content-type'] !== 'application/json') {
        error = 'Expected content-type of "application/json" for /search/events POST request. Received content-type: ' + req.headers['content-type'];
        console.error(error);
        return callback(error, {});
    }

    if (!req.body) {
        error = 'Empty body received for /search/events POST request.';
        console.error(error);
        return callback(error, {});
    }

    if (req.body.hasOwnProperty('type')) {
        // FIXME Hack to handle inconsistencies between photo and contacts collection
        if (req.body.type === 'photo') {
            req.body.type = 'photo/full';
        }
        // END FIXME

        var source = getSourceForEvent(req.body);

        if (!source) {
            error = 'No source found for event: '+JSON.stringify(req.body);
            console.error(error);
            return callback(error, {});
        }

        if (req.body.action === 'new' || req.body.action === 'update') {
            lsearch.indexTypeAndSource(req.body.type, source, req.body.obj.data, function(err, time) {
                if (err) {
                    handleError(req.body.type, req.body.action, req.body.obj.data._id, err);
                    return callback(err, {});
                }
                handleLog(req.body.type, req.body.action, req.body.obj.data._id, time);
                if (maxCloseTimeout) clearTimeout(maxCloseTimeout);
                maxCloseTimeout = setTimeout(function() {
                    lsearch.flushAndCloseWriter();
                }, MAX_CLOSE_TIMEOUT);
                return callback(err, {timeToIndex: time});
            });
        } else if (req.body.action === 'delete') {
            lsearch.deleteDocument(req.body.obj.data._id, function(err, time) {
                if (err) {
                    handleError(req.body.type, req.body.action, req.body.obj.data._id, err);
                    return callback(err, {});
                }
                handleLog(req.body.type, req.body.action, req.body.obj.data._id, time);
                return callback(err, {timeToIndex: time});
            });
        } else {
            console.log("Unexpected event: " + req.body.type + " and " + req.body.action);
            res.end();
        }
    } else {
        console.log("Unexpected event or not json " + req.headers["content-type"]);
        res.end();
    }
};

exports.handlePostIndex = function(req, callback) {
    var error;

    if (!req.body.type || !req.body.source || !req.body.data) {
        error = 'Invalid arguments given for /search/index POST request. '+JSON.stringify(req.body);
{'fraction_non_alphanumeric': 0.15346047094636941, 'fraction_numerical': 0.004476676515355001, 'mean_word_length': 2.3135568080688222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1179812', 'n_tokens_mistral': 3346, 'n_tokens_neox': 3231, 'n_words': 699}