<commit_before>var dgram = require("dgram");
var os = require("os");
var net = require("net");
var http = require("http");
var async = require("async");
var crypto = require("crypto");
var dns = require("dns");
var ursa = require("ursa"); // only need this to do the rsa encryption, not supported in crypto.*
var dhash = require("./dhash");

var REQUEST_TIMEOUT = 5 * 1000; // default timeout for any request
var warn = console.log; // switch to function(){} to disable
var debug = console.log; //function(){}; // switch to console.log to enable
var MESH_MAX = 200; // how many peers to maintain at most

var PEM_REGEX = /^(-----BEGIN (.*) KEY-----\r?\n[\/+=a-zA-Z0-9\r\n]*\r?\n-----END \2 KEY-----\r?\n)/m;

exports.hash = function(string)
{
  return new dhash.Hash(string);
}

// start a hashname listening and ready to go
exports.hashname = function(space, keys, args)
{
  if(!space || !keys || !keys.public || !keys.private) return undefined;
  if(!args) args = {};

  // configure defaults
  var self = {space:space, cb:{}, operators:[], watch:{}, lines:{}, lineq:[], seen:{}, buckets:[]};
  // parse/validate the private key
  self.prikey = keys.private;
  self.pubkey = keys.public;
  self.hash = new dhash.Hash(self.pubkey+space);
  self.hashname = self.hash.toString();
  if (!args.ip || args.natted) self.nat = true;
  self.ip = args.ip || "0.0.0.0";
  self.port = parseInt(args.port) || 0;

  // udp socket
  var counter = 1;
  self.server = dgram.createSocket("udp4", function(msg, rinfo){
    var packet = decode(msg);
    if(!packet) return warn("failed to decode a packet from", rinfo.address, rinfo.port, msg.toString());
    packet.from = {ip:rinfo.address, port:rinfo.port};
    packet.id = counter++;
    packet.at = Date.now();
    incoming(self, packet);
  });
  self.server.bind(self.port, self.ip, function(){
    // update address after listen completed to be besty
    self.port = self.server.address().port;
    self.address = [self.hashname, self.ip, self.port].join(",");
  });

  // try to set the correct address for logging, not important for telehash
  if(self.ip == "0.0.0.0") {
    var better;
    var ifaces = os.networkInterfaces()
    for (var dev in ifaces) {
      ifaces[dev].forEach(function(details){
        if(!better && details.family == "IPv4" && dev.substr(0,1) == "e") better = details.address;
      });
    }
    if(better) self.ip = better;
  }
  self.address = [self.hashname, self.ip, self.port].join(",");
  
  // try to resolve any dns-defined operators for this space
  dns.resolveSrv("_telehash._udp."+self.space, function(err, srvs){
    // if we didn't resolve anything, sometimes that's worth warning about
    if(err){
      if(!self.operator && self.operators.length === 0) warn("no operators found, couldn't resolve space", self.space, err.toString());
      return;
    }
    srvs.forEach(function(srv){
      var hashname = srv.name.split(".")[0];
      if(!dhash.isSHA1(hashname)) return warn("invalid operator address, not a hashname", srv);
      dns.resolve4(srv.name, function(err, ips){
        if(err) return warn("couldn't resolve operator", srv.name);
        ips.forEach(function(ip){
          var address = [hashname, ip, srv.port].join(",");
          seen(self, address).operator = true;
          debug("adding srv operator", address);
          self.operators.push(address);
        });
      });
    })
  });

  // set up methods (personal prefernce to do this explicitly vs. prototype pattern)
  self.myLookup = function(callback) {
    self.cb.lookup = callback;
    self.operator = true;
  };
  self.setOpen = function() {
    self.operator = true;
    self.open = true;
  };
  self.setOperators = function(addresses) {
    if(!Array.isArray(addresses)) return;
    self.operators = addresses.map(function(address){
      var op = seen(self, address);
      op.operator = true;
      return op.hashname;
    });
  };
  self.doVerify = function(hn, callback) { doVerify(self, hn, callback) };
  self.doLine = function(hn, callback) { doLine(self, hn, callback) };
  self.doStream = function(hn, callback) { return doStream(self, hn, callback) };
  self.doSocket = function(args, callback) { return doSocket(self, args, callback) };
  self.setProxy = function(args) {
    self.proxyBase = args;
  };
  self.doProxy = function(args, callback) { return doProxy(self, args, callback) };
  self.doProxyStream = function(args, callback) { return doProxyStream(self, args, callback) };

  self.setSeeds = function(addresses) {
    // set ourselves in DHT mode
    // line to the operators
    // have a 55sec timer to scan all lines
      // make bucket list
      // .see for closer to us for set amount in each bucket
    // have .see watcher for any new lines in empty buckets
  }

  // start the DHT meshing maintainence
  meshLoop(self);

  return self;
}

// every 25 seconds do the maintenance work for peers
function meshLoop(self)
{
  debug("MESHA")
  meshReap(self);
  meshSeen(self);
  meshElect(self);
  meshPing(self);
  debug("MESHZ")
  setTimeout(function(){meshLoop(self)}, 25*1000);
}

// delete any dead hashnames!
function meshReap(self)
{
  function del(who, why)
  {
    if(who.line) delete self.lines[who.line];
    delete self.seen[who];
    debug("reaping ", who, why);
  }
  Object.keys(self.seen).forEach(function(h){
    var hn = self.seen[h];
    if(!hn.sentAt) return; // TODO never if these are from app? remove old ones from .see hints?
    if(!hn.recvAt) {
      if(Date.now() - hn.at > 120*1000) return del(h, "sent, never received, older than 2min");
      return; // allow non-response for up to 2min
    }
    if(Date.now() - hn.sentAt > 60*1000) return del(h, "we stopped sending to them for more than 1min");
    if(hn.sentAt - hn.recvAt > 60*1000) return del(h, "no response in 30sec");
  });
}

// look for any newly seen hashnames to request a line to
function meshSeen(self)
{
  // scan the nearest 100
  var nearest;
  Object.keys(self.seen).map(function(h){return self.seen[h]}).sort(function(a, b){
    return self.hash.distanceTo(a.hash) - self.hash.distanceTo(b.hash);
  }).slice(0,100).forEach(function(hn){
    if(!nearest) nearest = hn;
    if(hn.open) return; // line is open or already tried opening a line to them at least once
    // try sending them a line request, no line == !hn.forApp
    doLine(self, hn);
  });<commit_msg>actually delete from line too<commit_after>var dgram = require("dgram");
var os = require("os");
var net = require("net");
var http = require("http");
var async = require("async");
var crypto = require("crypto");
var dns = require("dns");
var ursa = require("ursa"); // only need this to do the rsa encryption, not supported in crypto.*
var dhash = require("./dhash");

var REQUEST_TIMEOUT = 5 * 1000; // default timeout for any request
var warn = console.log; // switch to function(){} to disable
var debug = console.log; //function(){}; // switch to console.log to enable
var MESH_MAX = 200; // how many peers to maintain at most

var PEM_REGEX = /^(-----BEGIN (.*) KEY-----\r?\n[\/+=a-zA-Z0-9\r\n]*\r?\n-----END \2 KEY-----\r?\n)/m;

exports.hash = function(string)
{
  return new dhash.Hash(string);
}

// start a hashname listening and ready to go
exports.hashname = function(space, keys, args)
{
  if(!space || !keys || !keys.public || !keys.private) return undefined;
  if(!args) args = {};

  // configure defaults
  var self = {space:space, cb:{}, operators:[], watch:{}, lines:{}, lineq:[], seen:{}, buckets:[]};
  // parse/validate the private key
  self.prikey = keys.private;
  self.pubkey = keys.public;
  self.hash = new dhash.Hash(self.pubkey+space);
  self.hashname = self.hash.toString();
  if (!args.ip || args.natted) self.nat = true;
  self.ip = args.ip || "0.0.0.0";
  self.port = parseInt(args.port) || 0;

  // udp socket
  var counter = 1;
  self.server = dgram.createSocket("udp4", function(msg, rinfo){
    var packet = decode(msg);
    if(!packet) return warn("failed to decode a packet from", rinfo.address, rinfo.port, msg.toString());
    packet.from = {ip:rinfo.address, port:rinfo.port};
    packet.id = counter++;
    packet.at = Date.now();
    incoming(self, packet);
  });
  self.server.bind(self.port, self.ip, function(){
    // update address after listen completed to be besty
    self.port = self.server.address().port;
    self.address = [self.hashname, self.ip, self.port].join(",");
  });

  // try to set the correct address for logging, not important for telehash
  if(self.ip == "0.0.0.0") {
    var better;
    var ifaces = os.networkInterfaces()
    for (var dev in ifaces) {
      ifaces[dev].forEach(function(details){
        if(!better && details.family == "IPv4" && dev.substr(0,1) == "e") better = details.address;
      });
    }
    if(better) self.ip = better;
  }
  self.address = [self.hashname, self.ip, self.port].join(",");
  
  // try to resolve any dns-defined operators for this space
  dns.resolveSrv("_telehash._udp."+self.space, function(err, srvs){
    // if we didn't resolve anything, sometimes that's worth warning about
    if(err){
      if(!self.operator && self.operators.length === 0) warn("no operators found, couldn't resolve space", self.space, err.toString());
      return;
    }
    srvs.forEach(function(srv){
      var hashname = srv.name.split(".")[0];
      if(!dhash.isSHA1(hashname)) return warn("invalid operator address, not a hashname", srv);
      dns.resolve4(srv.name, function(err, ips){
        if(err) return warn("couldn't resolve operator", srv.name);
        ips.forEach(function(ip){
          var address = [hashname, ip, srv.port].join(",");
          seen(self, address).operator = true;
          debug("adding srv operator", address);
          self.operators.push(address);
        });
      });
    })
  });

  // set up methods (personal prefernce to do this explicitly vs. prototype pattern)
  self.myLookup = function(callback) {
    self.cb.lookup = callback;
    self.operator = true;
  };
  self.setOpen = function() {
    self.operator = true;
    self.open = true;
  };
  self.setOperators = function(addresses) {
    if(!Array.isArray(addresses)) return;
    self.operators = addresses.map(function(address){
      var op = seen(self, address);
      op.operator = true;
      return op.hashname;
    });
  };
  self.doVerify = function(hn, callback) { doVerify(self, hn, callback) };
  self.doLine = function(hn, callback) { doLine(self, hn, callback) };
  self.doStream = function(hn, callback) { return doStream(self, hn, callback) };
  self.doSocket = function(args, callback) { return doSocket(self, args, callback) };
  self.setProxy = function(args) {
    self.proxyBase = args;
  };
  self.doProxy = function(args, callback) { return doProxy(self, args, callback) };
  self.doProxyStream = function(args, callback) { return doProxyStream(self, args, callback) };

  self.setSeeds = function(addresses) {
    // set ourselves in DHT mode
    // line to the operators
    // have a 55sec timer to scan all lines
      // make bucket list
      // .see for closer to us for set amount in each bucket
    // have .see watcher for any new lines in empty buckets
  }

  // start the DHT meshing maintainence
  meshLoop(self);

  return self;
}

// every 25 seconds do the maintenance work for peers
function meshLoop(self)
{
  debug("MESHA")
  meshReap(self);
  meshSeen(self);
  meshElect(self);
  meshPing(self);
  debug("MESHZ")
  setTimeout(function(){meshLoop(self)}, 25*1000);
}

// delete any dead hashnames!
function meshReap(self)
{
  function del(who, why)
  {
    if(who.line) delete self.lines[who.line];
    delete self.seen[who.hashname];
    debug("reaping ", who.hashname, why);
  }
  Object.keys(self.seen).forEach(function(h){
    var hn = self.seen[h];
    if(!hn.sentAt) return; // TODO never if these are from app? remove old ones from .see hints?
    if(!hn.recvAt) {
      if(Date.now() - hn.at > 120*1000) return del(hn, "sent, never received, older than 2min");
      return; // allow non-response for up to 2min
    }
    if(Date.now() - hn.sentAt > 60*1000) return del(hn, "we stopped sending to them for more than 1min");
    if(hn.sentAt - hn.recvAt > 60*1000) return del(hn, "no response in 30sec");
  });
}

// look for any newly seen hashnames to request a line to
function meshSeen(self)
{
  // scan the nearest 100
  var nearest;
  Object.keys(self.seen).map(function(h){return self.seen[h]}).sort(function(a, b){
    return self.hash.distanceTo(a.hash) - self.hash.distanceTo(b.hash);
  }).slice(0,100).forEach(function(hn){
    if(!nearest) nearest = hn;
    if(hn.open) return; // line is open or already tried opening a line to them at least once
    // try sending them a line request, no line == !hn.forApp
    doLine(self, hn);
  });
{'fraction_non_alphanumeric': 0.15742845942767542, 'fraction_numerical': 0.01097608780870247, 'mean_word_length': 3.2705055239370604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '167600', 'n_tokens_mistral': 4286, 'n_tokens_neox': 4158, 'n_words': 1178}