<commit_before>/*jshint strict: false, unused: false */
/*global require, exports, ArangoServerState */

////////////////////////////////////////////////////////////////////////////////
/// @brief Cluster kickstarting functionality using dispatchers
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author <NAME>
/// @author Copyright 2014, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

// -----------------------------------------------------------------------------
// --SECTION--                                         Kickstarter functionality
// -----------------------------------------------------------------------------

var download = require("internal").download;
var executeExternal = require("internal").executeExternal;
var killExternal = require("internal").killExternal;
var statusExternal = require("internal").statusExternal;
var base64Encode = require("internal").base64Encode;
var yaml = require("js-yaml");

var fs = require("fs");
var wait = require("internal").wait;
var toArgv = require("internal").toArgv;

var exchangePort = require("org/arangodb/cluster/planner").exchangePort;
var exchangeProtocol = require("org/arangodb/cluster/planner").exchangeProtocol;

var console = require("console");

var launchActions = {};
var shutdownActions = {};
var cleanupActions = {};
var isHealthyActions = {};
var upgradeActions = {};

var getAddrPort = require("org/arangodb/cluster/planner").getAddrPort;
var getPort = require("org/arangodb/cluster/planner").getPort;
var endpointToURL = require("org/arangodb/cluster/planner").endpointToURL;

function extractErrorMessage (result) {
  if (result === undefined) {
    return "unknown error";
  }

  if (result.hasOwnProperty("body") && result.body !== undefined) {
    try {
      var e =  JSON.parse(result.body);

      if (e.hasOwnProperty("errorMessage")) {
        return e.errorMessage;
      }

      if (e.hasOwnProperty("errorNum")) {
        return e.errorNum;
      }

      if (result.body !== "") {
        return result.body;
      }
    }
    catch (err) {
      if (result.body !== "") {
        return result.body;
      }
    }
  }

  if (result.hasOwnProperty("message")) {
    return result.message;
  }

  if (result.hasOwnProperty("code")) {
    return String(result.code);
  }

  return "unknown error";
}

function makePath (path) {
  return fs.join.apply(null,path.split("/"));
}

function encode (st) {
  var st2 = "";
  var i;
  for (i = 0; i < st.length; i++) {
    if (st[i] === "_") {
      st2 += "@U";
    }
    else if (st[i] === "@") {
      st2 += "@@";
    }
    else {
      st2 += st[i];
    }
  }
  return encodeURIComponent(st2);
}

function getAuthorizationHeader (username, passwd) {
  return "Basic " + base64Encode(username + ":" + passwd);
}

function getAuthorization (dispatcher) {
  return getAuthorizationHeader(dispatcher.username, dispatcher.passwd);
}

function waitForServerUp (endpoint, timeout) {
  var url = endpointToURL(endpoint) + "/_api/version";
  var time = 0;
  while (true) {
    var r = download(url, "", {});
    if (r.code === 200 || r.code === 401) {
      console.info("Could talk to "+endpoint+" .");
      return true;
    }
    if (time >= timeout) {
      console.info("Could not talk to endpoint "+endpoint+", giving up.");
      return false;
    }
    wait(0.5);
    time += 0.5;
  }
}

function waitForServerDown (endpoint, timeout) {
  var url = endpointToURL(endpoint) + "/_api/version";
  var time = 0;
  while (true) {
    var r = download(url, "", {});
    if (r.code === 500 || r.code === 401) {
      console.info("Server at "+endpoint+" does not answer any more.");
      return true;
    }
    if (time >= timeout) {
      console.info("After timeout, server at "+endpoint+
                   "is still there, giving up.");
      return false;
    }
    wait(0.5);
    time += 0.5;
  }
}

function sendToAgency (agencyURL, path, obj) {
  var res;
  var body;

  console.info("Sending %s to agency...", path);
  if (typeof obj === "string") {
    var count = 0;
    while (count++ <= 2) {
      body = "value="+encodeURIComponent(obj);
      res = download(agencyURL+path,body,
          {"method":"PUT", "followRedirects": true,
           "headers": { "Content-Type": "application/x-www-form-urlencoded"}});
      if (res.code === 201 || res.code === 200) {
        return true;
      }
      wait(3);  // wait 3 seconds before trying again
    }
    return res;
  }
  if (typeof obj !== "object") {
    return "Strange object found: not a string or object";
  }
  var keys = Object.keys(obj);
  var i;
  if (keys.length !== 0) {
    for (i = 0; i < keys.length; i++) {
      res = sendToAgency (agencyURL, path+"/"+encode(keys[i]), obj[keys[i]]);
      if (res !== true) {
        return res;
      }
    }
    return true;
  }
  // Create a directory
  var count2 = 0;
  while (count2++ <= 2) {
    body = "dir=true";
    res = download(agencyURL+path,body,
          {"method": "PUT", "followRedirects": true,
           "headers": { "Content-Type": "application/x-www-form-urlencoded"}});
    if (res.code === 201 || res.code === 200) {
      return true;
    }
    wait(3);  // wait 3 seconds before trying again
  }
  return res;
}

launchActions.startAgent = function (dispatchers, cmd, isRelaunch) {
  console.info("Starting agent...");

  var dataPath = fs.makeAbsolute(cmd.dataPath);
  if (dataPath !== cmd.dataPath) {   // path was relative
    dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
  }

  var agentDataDir = fs.join(dataPath, "agent"+cmd.agencyPrefix+cmd.extPort);
  if (!isRelaunch) {
    if (fs.exists(agentDataDir)) {
      fs.removeDirectoryRecursive(agentDataDir,true);
    }
  }

  var instanceName = "agent"+cmd.agencyPrefix+cmd.extPort;
  var extBind = (cmd.onlyLocalhost ? "127.0.0.1:" : "0.0.0.0:")+cmd.extPort;
  var extEndpoint = getAddrPort(exchangePort(dispatchers[cmd.dispatcher].endpoint, cmd.extPort));
  var clusterBind = (cmd.onlyLocalhost ? "127.0.0.1:" : "0.0.0.0:")+cmd.intPort;
  var clusterEndPoint = getAddrPort(exchangePort(dispatchers[cmd.dispatcher].endpoint, cmd.intPort));

  var clusterUrl = "http://" + clusterBind;
  var agencyUrl =  "http://" + extBind;

  var args = {
      "data-dir":               agentDataDir,
      "name":                   instanceName,
      "bind-addr":              extBind,
      "addr":                   extEndpoint,
      "peer-bind-addr":         clusterBind,
      "peer-addr":              clusterEndPoint,
      "initial-cluster-state":  "new",
      "initial-cluster":        instanceName + "=" + clusterUrl<commit_msg>fixed endpoints<commit_after>/*jshint strict: false, unused: false */
/*global require, exports, ArangoServerState */

////////////////////////////////////////////////////////////////////////////////
/// @brief Cluster kickstarting functionality using dispatchers
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author <NAME>
/// @author Copyright 2014, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

// -----------------------------------------------------------------------------
// --SECTION--                                         Kickstarter functionality
// -----------------------------------------------------------------------------

var download = require("internal").download;
var executeExternal = require("internal").executeExternal;
var killExternal = require("internal").killExternal;
var statusExternal = require("internal").statusExternal;
var base64Encode = require("internal").base64Encode;
var yaml = require("js-yaml");

var fs = require("fs");
var wait = require("internal").wait;
var toArgv = require("internal").toArgv;

var exchangePort = require("org/arangodb/cluster/planner").exchangePort;
var exchangeProtocol = require("org/arangodb/cluster/planner").exchangeProtocol;

var console = require("console");

var launchActions = {};
var shutdownActions = {};
var cleanupActions = {};
var isHealthyActions = {};
var upgradeActions = {};

var getAddrPort = require("org/arangodb/cluster/planner").getAddrPort;
var getPort = require("org/arangodb/cluster/planner").getPort;
var endpointToURL = require("org/arangodb/cluster/planner").endpointToURL;

function extractErrorMessage (result) {
  if (result === undefined) {
    return "unknown error";
  }

  if (result.hasOwnProperty("body") && result.body !== undefined) {
    try {
      var e =  JSON.parse(result.body);

      if (e.hasOwnProperty("errorMessage")) {
        return e.errorMessage;
      }

      if (e.hasOwnProperty("errorNum")) {
        return e.errorNum;
      }

      if (result.body !== "") {
        return result.body;
      }
    }
    catch (err) {
      if (result.body !== "") {
        return result.body;
      }
    }
  }

  if (result.hasOwnProperty("message")) {
    return result.message;
  }

  if (result.hasOwnProperty("code")) {
    return String(result.code);
  }

  return "unknown error";
}

function makePath (path) {
  return fs.join.apply(null,path.split("/"));
}

function encode (st) {
  var st2 = "";
  var i;
  for (i = 0; i < st.length; i++) {
    if (st[i] === "_") {
      st2 += "@U";
    }
    else if (st[i] === "@") {
      st2 += "@@";
    }
    else {
      st2 += st[i];
    }
  }
  return encodeURIComponent(st2);
}

function getAuthorizationHeader (username, passwd) {
  return "Basic " + base64Encode(username + ":" + passwd);
}

function getAuthorization (dispatcher) {
  return getAuthorizationHeader(dispatcher.username, dispatcher.passwd);
}

function waitForServerUp (endpoint, timeout) {
  var url = endpointToURL(endpoint) + "/_api/version";
  var time = 0;
  while (true) {
    var r = download(url, "", {});
    if (r.code === 200 || r.code === 401) {
      console.info("Could talk to "+endpoint+" .");
      return true;
    }
    if (time >= timeout) {
      console.info("Could not talk to endpoint "+endpoint+", giving up.");
      return false;
    }
    wait(0.5);
    time += 0.5;
  }
}

function waitForServerDown (endpoint, timeout) {
  var url = endpointToURL(endpoint) + "/_api/version";
  var time = 0;
  while (true) {
    var r = download(url, "", {});
    if (r.code === 500 || r.code === 401) {
      console.info("Server at "+endpoint+" does not answer any more.");
      return true;
    }
    if (time >= timeout) {
      console.info("After timeout, server at "+endpoint+
                   "is still there, giving up.");
      return false;
    }
    wait(0.5);
    time += 0.5;
  }
}

function sendToAgency (agencyURL, path, obj) {
  var res;
  var body;

  console.info("Sending %s to agency...", path);
  if (typeof obj === "string") {
    var count = 0;
    while (count++ <= 2) {
      body = "value="+encodeURIComponent(obj);
      res = download(agencyURL+path,body,
          {"method":"PUT", "followRedirects": true,
           "headers": { "Content-Type": "application/x-www-form-urlencoded"}});
      if (res.code === 201 || res.code === 200) {
        return true;
      }
      wait(3);  // wait 3 seconds before trying again
    }
    return res;
  }
  if (typeof obj !== "object") {
    return "Strange object found: not a string or object";
  }
  var keys = Object.keys(obj);
  var i;
  if (keys.length !== 0) {
    for (i = 0; i < keys.length; i++) {
      res = sendToAgency (agencyURL, path+"/"+encode(keys[i]), obj[keys[i]]);
      if (res !== true) {
        return res;
      }
    }
    return true;
  }
  // Create a directory
  var count2 = 0;
  while (count2++ <= 2) {
    body = "dir=true";
    res = download(agencyURL+path,body,
          {"method": "PUT", "followRedirects": true,
           "headers": { "Content-Type": "application/x-www-form-urlencoded"}});
    if (res.code === 201 || res.code === 200) {
      return true;
    }
    wait(3);  // wait 3 seconds before trying again
  }
  return res;
}

launchActions.startAgent = function (dispatchers, cmd, isRelaunch) {
  console.info("Starting agent...");

  var dataPath = fs.makeAbsolute(cmd.dataPath);
  if (dataPath !== cmd.dataPath) {   // path was relative
    dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
  }

  var agentDataDir = fs.join(dataPath, "agent"+cmd.agencyPrefix+cmd.extPort);
  if (!isRelaunch) {
    if (fs.exists(agentDataDir)) {
      fs.removeDirectoryRecursive(agentDataDir,true);
    }
  }

  var instanceName = "agent"+cmd.agencyPrefix+cmd.extPort;
  var extEndpoint = getAddrPort(exchangePort(dispatchers[cmd.dispatcher].endpoint, cmd.extPort));
  var extBind = cmd.onlyLocalhost ? ("127.0.0.1:"+cmd.extPort) : extEndpoint;
  var clusterEndPoint = getAddrPort(exchangePort(dispatchers[cmd.dispatcher].endpoint, cmd.intPort));
  var clusterBind = cmd.onlyLocalhost ? ("127.0.0.1:"+cmd.intPort) : clusterEndPoint;

  var clusterUrl = "http://" + clusterEndPoint;
  var agencyUrl =  "http://" + extBind;

  var args = {
      "data-dir":               agentDataDir,
      "name":                   instanceName,
      "bind-addr":              extBind,
      "addr":                   extEndpoint,
      "peer-bind-addr":         clusterBind,
      "peer-addr":              clusterEndPoint,
      "initial-cluster-state":  "new",
      "initial-cluster":        instanceName + "=" + clusterUrl
{'fraction_non_alphanumeric': 0.19341227711991288, 'fraction_numerical': 0.011705458010072138, 'mean_word_length': 2.4142657992565058, 'pattern_counts': {'":': 34, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7316174', 'n_tokens_mistral': 4529, 'n_tokens_neox': 4355, 'n_words': 1116}