<commit_before>var http = require('http'),
    fs = require("fs"),
    mime = require("mime");
require('sugar');

// plugin loader
var plugins = require("./plugins")

// get args
var argv = require('minimist')(process.argv.slice(2));
var express = require('express')
var app = express();

// load plugins
var all = plugins.loadAll();

// query history
var history = []


// get a specific service's information
app.get("/api/service/:service", function(req, res, next) {

  // send the service's data, if possible
  if (  all.services[req.params.service]  ) {

    // get service data
    o = new all.services[req.params.service](all.services)

    // write out data, or error
    if (o.getData) {
      res.send( o.getData() || {} )
    } else {
      // no data
      var err = new Error();
      err.status = 501;
      next(err);
    }

  } else {
    // no service
    var err = new Error("Sorry, this service doesn't exist!");
    err.status = 404;
    next(err);
  }
});

// get all services
app.get("/api/services", function(req, res) {

  out = []
  Object.keys(all.services).each(function(name){
    // loop through each service

    if (typeof all.services[name] == "function") {
      srv = new all.services[name]();
    } else {
      srv = all.services[name]
    }

    // create record
    item = {name: name, title: name.capitalize(), html: null}

    if ( srv.getServicesPage ) {
      item.html = srv.getServicesPage()
    }
    out.push(item);
  })

  res.send(out);
})

// get a specific service's information
app.get("/api/history", function(req, res, next) {
  // get the query history
  res.send( {history: history} )
});

// post a query
app.post("/api/query", function(req, res, next) {
  var body = '';

  // a data chunk
  req.on('data', function(chunk) {
    body += chunk.toString();

    // Too much POST data, kill the connection!
    if (body.length > 1e6)
      req.connection.destroy();
  });

  // end of request
  req.on('end', function() {
    body = JSON.parse(body); // parse the body

    // create callback object
    var callbackObject = function(text, status, callback) {

      // failed request
      if (text == false) {
        res.send( {ERROR: status || null} )
      }

      // create packet, if it isn't already
      if (typeof text == "string") {
        packet = {"OK": text, "complete": callback == undefined}
      } else {
        packet = text;
      }

      // add to history
      hist = {packet: packet, when: new Date(), query: body, complete: callback == undefined, callback: callback, starus: status}
      history.push(hist)

      // end of query
      res.send(packet);
    }

    if (history.length == 0 || history[history.length-1].complete == true) {

      // sort all plugins based on priority
      all.all = all.all.sortBy(function(n) {
        return n.priority || 0
      })

      // do query
      resp = all.validateFor(body.query.text)

      // if query was successful, continue
      if (resp != false) {

        // get the plugin's response
        resp.then(body.query.text, all.services, callbackObject)

      } else {
        // no plugin's matched the query
        res.end( "{NOHIT: null}" )
      }

    } else {
      // otherwise, just run the callback
      hist = history[history.length-1]
      if (hist.status && hist.status.type == "boolean") {

        // do a boolean operation
        trueStuff = ["yes", "true", "yea", "yep", "yay", "correct"]
        falseStuff = ["no", "false", "nope", "ney", "nay", "incorrect"]

        ifTrue = body.query.text.split(' ').intersect(trueStuff).length > 0
        ifFalse = body.query.text.split(' ').intersect(falseStuff).length > 0

        if (ifTrue && !ifFalse) {
          hist.callback(true, null, callbackObject)
        } else if (!ifTrue && ifFalse) {
          hist.callback(false, null, callbackObject)
        } else {<commit_msg>added more words<commit_after>var http = require('http'),
    fs = require("fs"),
    mime = require("mime");
require('sugar');

// plugin loader
var plugins = require("./plugins")

// get args
var argv = require('minimist')(process.argv.slice(2));
var express = require('express')
var app = express();

// load plugins
var all = plugins.loadAll();

// query history
var history = []


// get a specific service's information
app.get("/api/service/:service", function(req, res, next) {

  // send the service's data, if possible
  if (  all.services[req.params.service]  ) {

    // get service data
    o = new all.services[req.params.service](all.services)

    // write out data, or error
    if (o.getData) {
      res.send( o.getData() || {} )
    } else {
      // no data
      var err = new Error();
      err.status = 501;
      next(err);
    }

  } else {
    // no service
    var err = new Error("Sorry, this service doesn't exist!");
    err.status = 404;
    next(err);
  }
});

// get all services
app.get("/api/services", function(req, res) {

  out = []
  Object.keys(all.services).each(function(name){
    // loop through each service

    if (typeof all.services[name] == "function") {
      srv = new all.services[name]();
    } else {
      srv = all.services[name]
    }

    // create record
    item = {name: name, title: name.capitalize(), html: null}

    if ( srv.getServicesPage ) {
      item.html = srv.getServicesPage()
    }
    out.push(item);
  })

  res.send(out);
})

// get a specific service's information
app.get("/api/history", function(req, res, next) {
  // get the query history
  res.send( {history: history} )
});

// post a query
app.post("/api/query", function(req, res, next) {
  var body = '';

  // a data chunk
  req.on('data', function(chunk) {
    body += chunk.toString();

    // Too much POST data, kill the connection!
    if (body.length > 1e6)
      req.connection.destroy();
  });

  // end of request
  req.on('end', function() {
    body = JSON.parse(body); // parse the body

    // create callback object
    var callbackObject = function(text, status, callback) {

      // failed request
      if (text == false) {
        res.send( {ERROR: status || null} )
      }

      // create packet, if it isn't already
      if (typeof text == "string") {
        packet = {"OK": text, "complete": callback == undefined}
      } else {
        packet = text;
      }

      // add to history
      hist = {packet: packet, when: new Date(), query: body, complete: callback == undefined, callback: callback, starus: status}
      history.push(hist)

      // end of query
      res.send(packet);
    }

    if (history.length == 0 || history[history.length-1].complete == true) {

      // sort all plugins based on priority
      all.all = all.all.sortBy(function(n) {
        return n.priority || 0
      })

      // do query
      resp = all.validateFor(body.query.text)

      // if query was successful, continue
      if (resp != false) {

        // get the plugin's response
        resp.then(body.query.text, all.services, callbackObject)

      } else {
        // no plugin's matched the query
        res.end( "{NOHIT: null}" )
      }

    } else {
      // otherwise, just run the callback
      hist = history[history.length-1]
      if (hist.status && hist.status.type == "boolean") {

        // do a boolean operation
        trueStuff = ["yes", "true", "yea", "yep", "yay", "correct", "ok", "sure"]
        falseStuff = ["no", "false", "nope", "ney", "nay", "incorrect"]

        ifTrue = body.query.text.split(' ').intersect(trueStuff).length > 0
        ifFalse = body.query.text.split(' ').intersect(falseStuff).length > 0

        if (ifTrue && !ifFalse) {
          hist.callback(true, null, callbackObject)
        } else if (!ifTrue && ifFalse) {
          hist.callback(false, null, callbackObject)
        } else {
{'fraction_non_alphanumeric': 0.1579961464354528, 'fraction_numerical': 0.0038535645472061657, 'mean_word_length': 2.3401973401973404, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125714', 'n_tokens_mistral': 2483, 'n_tokens_neox': 2388, 'n_words': 672}