<commit_before>require('rootpath')();
var logger = require('config/winston');

var Project = require('app/models/project');
var DocumentGroupAnnotation = require('app/models/document_group_annotation')
var ProjectInvitation = require('app/models/project_invitation')

var mongoose = require('mongoose');

// The project dashboard. Renders the projects page that lists all the projects of a user.
// Doesn't actually send any projects - that's done by 'getProjects', via AJAX.
module.exports.index = function(req, res) {
  // req.flash('success', "projects are cool");
  req.user.getProjects(function(err, projects) {
    if(err)
      res.send(err);
    else {
      res.render('projects', { title: "Projects" })
    }
  });  
}

// A function that returns all the projects of a user.
module.exports.getProjects = function(req, res) {
  req.user.getProjectsTableData(function(err, projects) {
    if(err) res.send(err);
    else {

      //setTimeout(function() {
      res.send({projects: projects});
      //}, 1);
    }
  });
}




// The tagging interface.
module.exports.tagging = function(req, res) {
  var id = req.params.id;
  var user = req.user;
  Project.findOne({ _id: id }, function(err, proj) {
    if(err) {
      res.send("error");
    }
    user.addProjectToRecentProjects(proj, function(err) {

      var canCreateNewCategories = user._id.equals(proj.user_id) || new Set(["full_permission", "create_edit_only"]).has(proj.category_hierarchy_permissions);
      var canDeleteCategories = user._id.equals(proj.user_id) || proj.category_hierarchy_permissions === "full_permission";

      proj.getDocumentGroupsPerUser(function(err, docGroupsPerUser) {
        if(err) { res.send("error"); }
        //console.log(canCreateNewCategories, canDeleteCategories, ">>>>>>>>>>>>>");
        res.render('tagging', { 
         projectName: proj.project_name,
         tagging: true,
         title: "Annotation Interface",
         numDocuments: docGroupsPerUser,
         canCreateNewCategories: canCreateNewCategories,
         canDeleteCategories: canDeleteCategories,
	       runDictionaryTagging: false// proj.user_id.equals(mongoose.Types.ObjectId("5ddcec0744a8f102041b524d"))
        });

      });

    });




  });
}





// Automatically tag all tokens on the screen that appear in the dictionary.
function runDictionaryTagging(documentGroup, dictionary) {
  // Build the automatic annotations based on the groupData.
  var automaticAnnotations = [];
  console.log(documentGroup)

  for(var doc_id = 0; doc_id < documentGroup.length; doc_id++) {
    var doc = documentGroup[doc_id];
    var alreadyFound = new Set();
    var anns = [];
      //console.log(doc)


    for(var ngram_size = 3; ngram_size >= 1; ngram_size--) {
      //console.log("NGRAM SIZE:", ngram_size)
      for(var i = 0; i < doc.length - ngram_size + 1; i++) {
        var ngram = doc.slice(i, i + ngram_size).join(" ")

        if(dictionary.hasOwnProperty(ngram)) {
          console.log(ngram, "is in the dictionary! Class:", dictionary[ngram])
          
          var start = i;
          var end = i + ngram_size;
          if(!alreadyFound.has(end)) {
            anns.push({start: start, end: end, labels: dictionary[ngram]});
            alreadyFound.add(end);
          }
        }


        //console.log('ngram:', ngram)
      }

    }
    automaticAnnotations.push(anns);<commit_msg>fixed bug<commit_after>require('rootpath')();
var logger = require('config/winston');

var Project = require('app/models/project');
var DocumentGroupAnnotation = require('app/models/document_group_annotation')
var ProjectInvitation = require('app/models/project_invitation')

var mongoose = require('mongoose');

// The project dashboard. Renders the projects page that lists all the projects of a user.
// Doesn't actually send any projects - that's done by 'getProjects', via AJAX.
module.exports.index = function(req, res) {
  // req.flash('success', "projects are cool");
  req.user.getProjects(function(err, projects) {
    if(err)
      res.send(err);
    else {
      res.render('projects', { title: "Projects" })
    }
  });  
}

// A function that returns all the projects of a user.
module.exports.getProjects = function(req, res) {
  req.user.getProjectsTableData(function(err, projects) {
    if(err) res.send(err);
    else {

      //setTimeout(function() {
      res.send({projects: projects});
      //}, 1);
    }
  });
}




// The tagging interface.
module.exports.tagging = function(req, res) {
  var id = req.params.id;
  var user = req.user;
  Project.findOne({ _id: id }, function(err, proj) {
    if(err) {
      res.send("error");
    }
    user.addProjectToRecentProjects(proj, function(err) {

      var canCreateNewCategories = user._id.equals(proj.user_id) || new Set(["full_permission", "create_edit_only"]).has(proj.category_hierarchy_permissions);
      var canDeleteCategories = user._id.equals(proj.user_id) || proj.category_hierarchy_permissions === "full_permission";

      proj.getDocumentGroupsPerUser(function(err, docGroupsPerUser) {
        if(err) { res.send("error"); }
        //console.log(canCreateNewCategories, canDeleteCategories, ">>>>>>>>>>>>>");
        res.render('tagging', { 
         projectName: proj.project_name,
         tagging: true,
         title: "Annotation Interface",
         numDocuments: docGroupsPerUser,
         canCreateNewCategories: canCreateNewCategories,
         canDeleteCategories: canDeleteCategories,
	       runDictionaryTagging: false// proj.user_id.equals(mongoose.Types.ObjectId("5ddcec0744a8f102041b524d"))
        });

      });

    });




  });
}





// Automatically tag all tokens on the screen that appear in the dictionary.
function runDictionaryTagging(documentGroup, dictionary) {
  // Build the automatic annotations based on the groupData.
  var automaticAnnotations = [];
  console.log(documentGroup)

  for(var doc_id = 0; doc_id < documentGroup.length; doc_id++) {
    var doc = documentGroup[doc_id];
    var anns = [];
      //console.log(doc)

    labeledTokens = new Array(doc.length);
    for(var x = i; x < labeledTokens.length; x++) {
      labeledTokens[x] = 0;              
    }


    for(var ngram_size = 3; ngram_size >= 1; ngram_size--) {
      //console.log("NGRAM SIZE:", ngram_size)
      for(var i = 0; i < doc.length - ngram_size + 1; i++) {
        var ngram = doc.slice(i, i + ngram_size).join(" ")

        if(dictionary.hasOwnProperty(ngram)) {
          console.log(ngram, "is in the dictionary! Class:", dictionary[ngram])
          
          var start = i;
          var end = i + ngram_size;
          alreadyLabeled = false;
          for(var x = i; x < end; x++) {
            if(labeledTokens[x] > 0) {
              alreadyLabeled = true;
            }
          }

          if(!alreadyLabeled) {
            anns.push({start: start, end: end, labels: dictionary[ngram]});
            for(var x = i; x < end; x++) {
              labeledTokens[x] = 1;              
            }
          }
        }


        //console.log('ngram:', ngram)
      }

    }
    automaticAnnotations.push(anns);
{'fraction_non_alphanumeric': 0.1379794857383729, 'fraction_numerical': 0.006322888857664746, 'mean_word_length': 2.883251500272777, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3730702', 'n_tokens_mistral': 2301, 'n_tokens_neox': 2173, 'n_words': 474}