<commit_before>var yaml = require('js-yaml'),
    fs   = require('fs'),
    mongoosastic = require('mongoosastic'),
    File = require('../models/composeFiles.js'),
    User = require('../models/users.js');

var auth = function(req, res, next){
    if (req.isAuthenticated()) {
         return next();
    } else {
        res.redirect('/');
    }
};

function tokenizer(name){
    var newArray = [];
    var array = name.split('');
    newArray[0] = array[0];
    for(var i = 1; i < array.length; i++){
        newArray[i] = newArray[i-1]+array[i];
    }
    return newArray;
}

var stream = File.find().stream();
var total = 0;
var count = 0;
var done = false;

File.createMapping(function(err, mapping){
  if(err){
    console.log('error creating mapping (you can safely ignore this)');
    console.log(err);
  } else {
    console.log('mapping created!');
    console.log(mapping);
  }
});

function reIndex(stream, count, total, done){
  stream.on('data', function(doc){
    total++;
    count++;
    doc.index(function() {
      count--;
      allDone();
    });
  });

  stream.on('close', function() {
    done = true;
    allDone();
  });

  stream.on('error', function(err){
    throw err;
  });
}

function allDone() {
  if (count > 0) return;
  if (!done) return;
  console.log('indexed ' + total + ' docs');
  process.exit(0);
}

module.exports = function(app) {
    app.post('/api/v1/create', auth, function(req, res){
        var serviceTags = [];
        var images = [];

        for(var key in req.body.params.form.stackfile){
            serviceTags.push(key);
            images.push(req.body.params.form.stackfile[key].image);
        }

        var file = new File({
            title: req.body.params.form.title,
            stackfile: req.body.params.form.stackfile,
            branch: req.body.params.form.branch,
            path : req.body.params.form.path,
            user: req.body.params.form.orgname,
            author: req.user.username,
            token: tokenizer(req.body.params.form.title),
            profileLink: "https://github.com/"+req.body.params.form.orgname,
            projectName: req.body.params.form.name,
            tags: serviceTags,
            images: images
        });

        file.save(function(err, savedFile){
            if(err){
                res.send(err);
            }
            file.on('es-indexed', function(){
                console.log('file indexed');
            });
        });
        res.redirect('/registry/' + file._id);
    });

    app.get('/api/v1/files', function(req, res, next){<commit_msg>enabled tokenizer on service tags<commit_after>var yaml = require('js-yaml'),
    fs   = require('fs'),
    mongoosastic = require('mongoosastic'),
    File = require('../models/composeFiles.js'),
    User = require('../models/users.js');

var auth = function(req, res, next){
    if (req.isAuthenticated()) {
         return next();
    } else {
        res.redirect('/');
    }
};

function tokenizer(name){
    var newArray = [];
    var array = name.split('');
    newArray[0] = array[0];
    for(var i = 1; i < array.length; i++){
        newArray[i] = newArray[i-1]+array[i];
    }
    return newArray;
}

var stream = File.find().stream();
var total = 0;
var count = 0;
var done = false;

File.createMapping(function(err, mapping){
  if(err){
    console.log('error creating mapping (you can safely ignore this)');
    console.log(err);
  } else {
    console.log('mapping created!');
    console.log(mapping);
  }
});

function reIndex(stream, count, total, done){
  stream.on('data', function(doc){
    total++;
    count++;
    doc.index(function() {
      count--;
      allDone();
    });
  });

  stream.on('close', function() {
    done = true;
    allDone();
  });

  stream.on('error', function(err){
    throw err;
  });
}

function allDone() {
  if (count > 0) return;
  if (!done) return;
  console.log('indexed ' + total + ' docs');
  process.exit(0);
}

module.exports = function(app) {
    app.post('/api/v1/create', auth, function(req, res){
        var serviceTags = [];
        var images = [];

        for(var key in req.body.params.form.stackfile){
            serviceTags.push(tokenizer(key));
            images.push(req.body.params.form.stackfile[key].image);
        }

        var file = new File({
            title: req.body.params.form.title,
            stackfile: req.body.params.form.stackfile,
            branch: req.body.params.form.branch,
            path : req.body.params.form.path,
            user: req.body.params.form.orgname,
            author: req.user.username,
            token: tokenizer(req.body.params.form.title),
            profileLink: "https://github.com/"+req.body.params.form.orgname,
            projectName: req.body.params.form.name,
            tags: serviceTags,
            images: images
        });

        file.save(function(err, savedFile){
            if(err){
                res.send(err);
            }
            file.on('es-indexed', function(){
                console.log('file indexed');
            });
        });
        res.redirect('/registry/' + file._id);
    });

    app.get('/api/v1/files', function(req, res, next){
{'fraction_non_alphanumeric': 0.1693936477382098, 'fraction_numerical': 0.0038498556304138597, 'mean_word_length': 2.201478743068392, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4961827', 'n_tokens_mistral': 1655, 'n_tokens_neox': 1624, 'n_words': 321}