<commit_before>var async            = require('async');
var _                = require('underscore');
var sha1             = require('sha1');
var merkle           = require('merkle');
var vucoin           = require('vucoin');
var eventStream      = require('event-stream');
var inquirer         = require('inquirer');
var dos2unix         = require('./dos2unix');
var parsers          = require('./streams/parsers/doc');
var extractSignature = require('./streams/extractSignature');
var localValidator   = require('./localValidator');
var logger           = require('./logger')('sync');

var CONST_FORCE_TX_PROCESSING = false;

module.exports = function Synchroniser (server, host, port, conf) {
  var that = this;

  // Services
  var TransactionService = server.TransactionsService;
  var PeeringService     = server.PeeringService;
  var ParametersService  = server.ParametersService;
  var BlockchainService  = server.BlockchainService;

  // Models
  var Block      = server.conn.model('Block');
  var Merkle        = server.conn.model('Merkle');
  var Membership    = server.conn.model('Membership');
  var Transaction   = server.conn.model('Transaction');
  var Peer          = server.conn.model('Peer');
  var Configuration = server.conn.model('Configuration');
  
  this.remoteFingerprint = null;

  this.sync = function (done) {
    logger.info('Connecting remote host...');
    vucoin(host, port, function (err, node) {
      if(err){
        done('Cannot sync: ' + err);
        return;
      }

      // Global sync vars
      var remotePeer = new Peer({});
      var remoteJsonPeer = {};
      var remoteCurrentNumber;
      var remotePubkey;

      async.waterfall([
        function (next){
          logger.info('Sync started.');
          next();
        },

        //============
        // Peer
        //============
        function (next){
          node.network.peering.get(next);
        },
        function (json, next){
          remotePubkey = json.pubkey;
          remotePeer.copyValuesFrom(json);
          var entry = remotePeer.getRaw();
          var signature = dos2unix(remotePeer.signature);
          // Parameters
          if(!(entry && signature)){
            callback('Requires a peering entry + signature');
            return;
          }

          remoteJsonPeer = json;
          remoteJsonPeer.pub = json.pubkey;
          remoteJsonPeer.status = "NOTHING";
          localValidator().checkPeerSignature(remoteJsonPeer, next);
        },
        function (next) {
          async.waterfall([
            function (next){
              PeeringService.submit(remoteJsonPeer, next);
            },
          ], function (err) {
            next(err);
          });
        },

        //============
        // Blockchain
        //============
        function (next){
          logger.info('Downloading Blockchain...');
          node.blockchain.current(function (err, current) {
            next(null, err ? null : current);
          });
        },
        function (current, next) {
          var i = 0;
          async.whilst(
            function () { return current ? i <= current.number : false; },
            function (callback) {
                async.waterfall([
                  function (next) {
                    node.blockchain.block(i, next);
                  },
                  function (block, next) {
                    BlockchainService.submitBlock(block, next);
                  },
                  function (block, next) {
                    i++;
                    next();
                  }
                ], callback);
            }, next);
        },

        //==============
        // Transactions
        //==============
        // function (next){
        //   Key.find({ managed: true }, next);
        // },
        // function (keys, next) {
        //   async.forEachSeries(keys, function (key, onKeyDone) {
        //     syncTransactionsOfKey(node, key.fingerprint, onKeyDone);
        //   }, next);
        // },

        //=======
        // Peers
        //=======<commit_msg>Fix: sync: now continues download instead of retrying from B#0<commit_after>var async            = require('async');
var _                = require('underscore');
var sha1             = require('sha1');
var merkle           = require('merkle');
var vucoin           = require('vucoin');
var eventStream      = require('event-stream');
var inquirer         = require('inquirer');
var dos2unix         = require('./dos2unix');
var parsers          = require('./streams/parsers/doc');
var extractSignature = require('./streams/extractSignature');
var localValidator   = require('./localValidator');
var logger           = require('./logger')('sync');

var CONST_FORCE_TX_PROCESSING = false;

module.exports = function Synchroniser (server, host, port, conf) {
  var that = this;

  // Services
  var TransactionService = server.TransactionsService;
  var PeeringService     = server.PeeringService;
  var ParametersService  = server.ParametersService;
  var BlockchainService  = server.BlockchainService;

  // Models
  var Block      = server.conn.model('Block');
  var Merkle        = server.conn.model('Merkle');
  var Membership    = server.conn.model('Membership');
  var Transaction   = server.conn.model('Transaction');
  var Peer          = server.conn.model('Peer');
  var Configuration = server.conn.model('Configuration');
  
  this.remoteFingerprint = null;

  this.sync = function (done) {
    logger.info('Connecting remote host...');
    vucoin(host, port, function (err, node) {
      if(err){
        done('Cannot sync: ' + err);
        return;
      }

      // Global sync vars
      var remotePeer = new Peer({});
      var remoteJsonPeer = {};
      var remoteCurrentNumber;
      var remotePubkey;

      async.waterfall([
        function (next){
          logger.info('Sync started.');
          next();
        },

        //============
        // Peer
        //============
        function (next){
          node.network.peering.get(next);
        },
        function (json, next){
          remotePubkey = json.pubkey;
          remotePeer.copyValuesFrom(json);
          var entry = remotePeer.getRaw();
          var signature = dos2unix(remotePeer.signature);
          // Parameters
          if(!(entry && signature)){
            callback('Requires a peering entry + signature');
            return;
          }

          remoteJsonPeer = json;
          remoteJsonPeer.pub = json.pubkey;
          remoteJsonPeer.status = "NOTHING";
          localValidator().checkPeerSignature(remoteJsonPeer, next);
        },
        function (next) {
          async.waterfall([
            function (next){
              PeeringService.submit(remoteJsonPeer, next);
            },
          ], function (err) {
            next(err);
          });
        },

        //============
        // Blockchain
        //============
        function (next){
          logger.info('Downloading Blockchain...');
          async.parallel({
            localCurrent: function (next) {
              BlockchainService.current(next);
            },
            remoteCurrent: function (next) {
              node.blockchain.current(function (err, current) {
                next(null, err ? null : current);
              });
            }
          }, next);
        },
        function (res, next) {
          var lCurrent = res.localCurrent;
          var rCurrent = res.remoteCurrent;
          var i = lCurrent ? lCurrent.number + 1 : 0;
          async.whilst(
            function () { return rCurrent ? i <= rCurrent.number : false; },
            function (callback) {
                async.waterfall([
                  function (next) {
                    node.blockchain.block(i, next);
                  },
                  function (block, next) {
                    BlockchainService.submitBlock(block, next);
                  },
                  function (block, next) {
                    i++;
                    next();
                  }
                ], callback);
            }, next);
        },

        //==============
        // Transactions
        //==============
        // function (next){
        //   Key.find({ managed: true }, next);
        // },
        // function (keys, next) {
        //   async.forEachSeries(keys, function (key, onKeyDone) {
        //     syncTransactionsOfKey(node, key.fingerprint, onKeyDone);
        //   }, next);
        // },

        //=======
        // Peers
        //=======
{'fraction_non_alphanumeric': 0.14442755233306045, 'fraction_numerical': 0.0016372354110630336, 'mean_word_length': 1.6699968779269434, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6090741', 'n_tokens_mistral': 2234, 'n_tokens_neox': 2163, 'n_words': 473}