<commit_before>module.exports = function(app) {

    var stackWhoConfig = require('./common/config.js');
    var dbUrl = stackWhoConfig.dbEndpoint;

    var nano = require('nano')(dbUrl);
    var dbName = 'test';
    var db = nano.use(dbName);

    var ChunkFetcher = require('./chunkFetcher/chunkFetcher.js');
    var CouchDbStore = require('./chunkFetcher/couchdbStore.js');
    var userTagInterceptor = require('./interceptor/userTagInterceptor.js');
    var Lexer = require('./lexer.js');
    var UserFilter = require('./userFilter.js');
    var https = require('https');
    var unicodeEnd = '%EF%BF%B0'; //\ufff0

    var users = [];
    var state = 'booting';

    var lexer = new Lexer();
    var userFilter = new UserFilter();

    //it's lame to have that here. We should find a different solution
    var designDoc =     {
                             "language": "javascript",
                             "views": {
                                 "by_location": {
                                     "map": "function(doc) { \n  if (doc.location != null){\n    emit(doc.location.toLowerCase(), doc);\n    doc.location.split(/\\W+/).forEach(function(word){ \n      emit(word.toLowerCase(), doc) \n    }); \n  } \n}"
                                 },
                                 "by_reputation": {
                                     "map": "function(doc) { if (doc.reputation != null) emit(doc.reputation, doc) }"
                                 },
                                 "by_location_tags": {
                                     "map": "function(doc) { if (doc.top_tags) { for(i=0;i<doc.top_tags.length;i++) { emit([doc.top_tags[i].tag_name, doc.location], doc); } } }"
                                 }
                             }
                         };


    var isValid = function(request, response){
        if (request.query.pw !== stackWhoConfig.adminPassword){
            response.send('wrong password');
            return false;
        }

        return true;
    };

    app.get('/rebuildIndex', function(request, response) {

        if(!isValid(request, response)){
            return;
        }

        response.send('rebuilding index...');

        // clean up the database we created previously
        nano.db.destroy(dbName, function() {
              // create a new database
            nano.db.create(dbName, function() {

                //add the design document containing our views
                db.insert(designDoc, '_design/userViews');

                new ChunkFetcher({
                    url: 'http://api.stackoverflow.com/1.1/users?',
                    key: 'users',
                    pageSize: 100,
                    maxLength: 20000,
                    interceptor: userTagInterceptor,
                    store: CouchDbStore
                })
                .fetch()
                .then(function(users){
                    console.log(users);
                });
            });
        });
    });
    
    app.get('/resumeIndexBuild', function(request, response) {

        if(!isValid(request, response)){
            return;
        }

        //get the user where we left off<commit_msg>chore(server): removed design doc as we now query in-memory<commit_after>module.exports = function(app) {

    var stackWhoConfig = require('./common/config.js');
    var dbUrl = stackWhoConfig.dbEndpoint;

    var nano = require('nano')(dbUrl);
    var dbName = 'test';
    var db = nano.use(dbName);

    var ChunkFetcher = require('./chunkFetcher/chunkFetcher.js');
    var CouchDbStore = require('./chunkFetcher/couchdbStore.js');
    var userTagInterceptor = require('./interceptor/userTagInterceptor.js');
    var Lexer = require('./lexer.js');
    var UserFilter = require('./userFilter.js');
    var https = require('https');
    var unicodeEnd = '%EF%BF%B0'; //\ufff0

    var users = [];
    var state = 'booting';

    var lexer = new Lexer();
    var userFilter = new UserFilter();

    var isValid = function(request, response){
        if (request.query.pw !== stackWhoConfig.adminPassword){
            response.send('wrong password');
            return false;
        }

        return true;
    };

    app.get('/rebuildIndex', function(request, response) {

        if(!isValid(request, response)){
            return;
        }

        response.send('rebuilding index...');

        // clean up the database we created previously
        nano.db.destroy(dbName, function() {
              // create a new database
            nano.db.create(dbName, function() {

                new ChunkFetcher({
                    url: 'http://api.stackoverflow.com/1.1/users?',
                    key: 'users',
                    pageSize: 100,
                    maxLength: 20000,
                    interceptor: userTagInterceptor,
                    store: CouchDbStore
                })
                .fetch()
                .then(function(users){
                    console.log(users);
                });
            });
        });
    });
    
    app.get('/resumeIndexBuild', function(request, response) {

        if(!isValid(request, response)){
            return;
        }

        //get the user where we left off
{'fraction_non_alphanumeric': 0.13065902578796562, 'fraction_numerical': 0.004775549188156638, 'mean_word_length': 1.5591397849462365, 'pattern_counts': {'":': 8, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4573966', 'n_tokens_mistral': 1449, 'n_tokens_neox': 1376, 'n_words': 302}