<commit_before>// dialects/http.js
// Provide service endpoints for pushing to services in qb

// vendor
var _ = require('underscore'),
  express = require('express'),
  request = require('request');

module.exports = {
  name: 'http',
  type: 'rpc',
  startup: startup,
}

function HttpDialect(qb, options) {
  this.qb = qb;
  this.options = options;
  this.auth = options.auth;
  this.retry = options.retry === undefined ? 0 : options.retry * 1;
  this.types = {};
  this.app = create_app(qb, options, this.types);
  this.ended = false;
}

HttpDialect.prototype.can = function() {
  var newtypes = Array.prototype.slice.call(arguments),
    types = this.types;

  _.each(newtypes, function (type) {
    types[type] = true;
  });
}

HttpDialect.prototype.push = function (endpoint, type, task, callback) {
  if (this.ended)
    return callback(new Error('qb ended'))
  make_request(endpoint, type, task, this.retry, this.auth, callback);
}

HttpDialect.prototype.end = function (cb) {
  this.ended = true;
  if (this.app) {
    this.app.close(cb)
  }
  else
    cb();
}

function startup(qb, options) {
  return new HttpDialect(qb, options);
}

function create_app(qb, options, types) {
  var base = options.base || '',
    port = options.port,
    app = options.app || express()
      .use(express.logger())
      .use(express.query())
      .use(express.json())
      .use(express.urlencoded()),
    hasApp = Boolean(options.app);

  if (hasApp) {
    qb.log.info('Staring http qb api server on existing app at %s', base)
  } else {
    qb.log.info('Starting http qb api server at :%d%s', port, base)
  }

  // If no port is given, we assume they don't want to listen
  if (!hasApp && !port) return null;

  if (options.auth) {
    qb.log.info('Using basic auth in http.')
    app.use(base, express.basicAuth(options.auth.user, options.auth.pass));
  }

  app.use(base, getTypeCallback(types))
    .use(base, pushEndpoint(qb));

  if (!hasApp) {
    return app.listen(port);
  }
  return null;
}

function make_request(endpoint, type, task, nretries, auth, callback) {
  // Here's a good way to do
  request({
    method: 'POST',
    uri: endpoint + '/' + type,
    json: task,
    auth: auth
  }, function (err, resp) {
    if (err && nretries > 0) {
      return make_request(endpoint, type, task, nretries - 1, callback);
    } else if (!resp || resp.statusCode !== 200) {
      err = new Error(JSON.stringify(resp ? resp.body : 'no response'));
    }
    callback(err, resp);
  });
}

function verifyBaseUrl(base) {
  return function (req, res, next) {
    // Set the base url
    if (req.path.slice(0, base.length) === base) {
      return next();
    }
    res.send(404, {error: 'all routes begin with ' + base});
  }<commit_msg>Make sure app is accessible and is an express server in all cases<commit_after>// dialects/http.js
// Provide service endpoints for pushing to services in qb

// vendor
var _ = require('underscore'),
  express = require('express'),
  request = require('request');

module.exports = {
  name: 'http',
  type: 'rpc',
  startup: startup,
}

function HttpDialect(qb, options) {
  this.qb = qb;
  this.options = options;
  this.auth = options.auth;
  this.retry = options.retry === undefined ? 0 : options.retry * 1;
  this.types = {};
  this.app = create_app(qb, options, this.types);
  this.ended = false;

  if (!options.app && options.port) {
    this.server = this.app.listen(options.port)
  }
}

HttpDialect.prototype.can = function() {
  var newtypes = Array.prototype.slice.call(arguments),
    types = this.types;

  _.each(newtypes, function (type) {
    types[type] = true;
  });
}

HttpDialect.prototype.push = function (endpoint, type, task, callback) {
  if (this.ended)
    return callback(new Error('qb ended'))
  make_request(endpoint, type, task, this.retry, this.auth, callback);
}

HttpDialect.prototype.end = function (cb) {
  this.ended = true;
  if (this.server) {
    this.server.close(cb)
  }
  else
    cb();
}

function startup(qb, options) {
  return new HttpDialect(qb, options);
}

function create_app(qb, options, types) {
  var base = options.base || '',
    port = options.port,
    app = options.app || express()
      .use(express.logger())
      .use(express.query())
      .use(express.json())
      .use(express.urlencoded()),
    hasApp = Boolean(options.app);

  if (hasApp) {
    qb.log.info('Staring http qb api server on existing app at %s', base)
  } else {
    qb.log.info('Starting http qb api server at :%d%s', port, base)
  }

  if (options.auth) {
    qb.log.info('Using basic auth in http.')
    app.use(base, express.basicAuth(options.auth.user, options.auth.pass));
  }

  app.use(base, getTypeCallback(types))
    .use(base, pushEndpoint(qb));

  return app
}

function make_request(endpoint, type, task, nretries, auth, callback) {
  // Here's a good way to do
  request({
    method: 'POST',
    uri: endpoint + '/' + type,
    json: task,
    auth: auth
  }, function (err, resp) {
    if (err && nretries > 0) {
      return make_request(endpoint, type, task, nretries - 1, callback);
    } else if (!resp || resp.statusCode !== 200) {
      err = new Error(JSON.stringify(resp ? resp.body : 'no response'));
    }
    callback(err, resp);
  });
}

function verifyBaseUrl(base) {
  return function (req, res, next) {
    // Set the base url
    if (req.path.slice(0, base.length) === base) {
      return next();
    }
    res.send(404, {error: 'all routes begin with ' + base});
  }
{'fraction_non_alphanumeric': 0.15549059016992509, 'fraction_numerical': 0.004019733235885255, 'mean_word_length': 3.1595744680851063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '689313', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1870, 'n_words': 488}