<commit_before>var async = require('async')
var util = require('util')
var events = require('events')
var _ = require('lodash')
var CCTransaction = require('cc-transaction')
var bitcoin = require('bitcoin-async')
var get_assets_outputs = require('cc-get-assets-outputs')

var properties
var bitcoin_rpc
var debug

function Scanner (settings, db) {
  // console.log('Mongoose', db)
  var self = this

  debug = settings.debug
  self.to_revert = []
  self.priority_parse_list = []
  properties = settings.properties
  self.next_hash = settings.next_hash
  self.last_hash = settings.last_hash
  self.last_block = settings.last_block
  self.last_fully_parsed_block = settings.last_fully_parsed_block
  bitcoin_rpc = new bitcoin.Client(settings.rpc_settings)

  self.Blocks = db.model('blocks', require(__dirname + '/models/blocks')(db, properties))
  self.RawTransactions = db.model('rawtransactions', require(__dirname + '/models/rawtransactions')(db, properties))
  self.Utxo = db.model('utxo', require(__dirname + '/models/utxo')(db))
  self.AddressesTransactions = db.model('addressestransactions', require(__dirname + '/models/addressestransactions')(db))
  self.AddressesUtxos = db.model('addressesutxos', require(__dirname + '/models/addressesutxos')(db))
  self.AssetsTransactions = db.model('assetstransactions', require(__dirname + '/models/assetstransactions')(db))
  self.AssetsUtxos = db.model('assetsutxos', require(__dirname + '/models/assetsutxos')(db))
  self.AssetsAddresses = db.model('assetsaddresses', require(__dirname + '/models/assetsaddresses')(db))

  if (process.env.ROLE === properties.roles.SCANNER) {
    self.on('newblock', function (newblock) {
      process.send({to: properties.roles.API, newblock: newblock})
    })
    self.on('newtransaction', function (newtransaction) {
      process.send({to: properties.roles.API, newtransaction: newtransaction})
    })
    self.on('newcctransaction', function (newcctransaction) {
      process.send({to: properties.roles.API, newcctransaction: newcctransaction})
    })
    self.on('revertedblock', function (revertedblock) {
      process.send({to: properties.roles.API, revertedblock: revertedblock})
    })
    self.on('revertedtransaction', function (revertedtransaction) {
      process.send({to: properties.roles.API, revertedtransaction: revertedtransaction})
    })
    self.on('revertedcctransaction', function (revertedcctransaction) {
      process.send({to: properties.roles.API, revertedcctransaction: revertedcctransaction})
    })
    self.on('mempool', function () {
      process.send({to: properties.roles.API, mempool: true})
    })
  }

  self.mempool_cargo = async.cargo(function (tasks, callback) {
    self.parse_mempool_cargo(tasks, callback)
  }, 500)
}

util.inherits(Scanner, events.EventEmitter)

Scanner.prototype.scan_blocks = function (err) {
  var self = this
  if (err) {
    console.error(err)
    return self.scan_blocks()
  }
  var job
  var next_block
  var last_hash

  async.waterfall([
    function (cb) {
      self.get_next_new_block(cb)
    },
    function (l_next_block, l_last_hash, cb) {
      last_hash = l_last_hash
      next_block = l_next_block || 0
      self.get_raw_block(next_block, cb)
    },
    function (raw_block_data, cb) {
      if (!cb) {
        cb = raw_block_data
        raw_block_data = null
      }
      if (!raw_block_data || (raw_block_data.height === next_block - 1 && raw_block_data.hash === last_hash)) {
        setTimeout(function () {
          if (debug) {
            job = 'mempool_scan'
            console.time(job)
          }
          self.parse_new_mempool(cb)
        }, 500)
      } else if (!raw_block_data.previousblockhash || raw_block_data.previousblockhash === last_hash) {
        // logger.debug('parsing block')
        if (debug) {
          job = 'parse_new_block'
          console.time(job)
        }
        self.parse_new_block(raw_block_data, cb)
      } else {
        if (debug) {
          job = 'reverting_block'
          console.time(job)
        }
        self.revert_block(next_block - 1, cb)
      }
    }
  ], function (err) {
    if (debug && job) console.timeEnd(job)
    if (err) self.to_revert = []
    self.scan_blocks(err)
  })
}

Scanner.prototype.revert_block = function (block_height, callback) {
  var self = this
  console.log('Reverting block: ' + block_height)
  var conditions = {
    height: block_height
  }
  self.Blocks.findOne(conditions).exec(function (err, block_data) {
    if (err) return callback(err)
    if (!block_data || !block_data.tx) return callback()
    var block_id = {
      height: block_data.height,
      hash: block_data.hash
    }
    var utxo_bulk = self.Utxo.collection.initializeOrderedBulkOp()
    // var holders_bulk = Holders.collection.initializeOrderedBulkOp()
    var addresses_transactions_bulk = self.AddressesTransactions.collection.initializeOrderedBulkOp()
    var addresses_utxos_bulk = self.AddressesUtxos.collection.initializeOrderedBulkOp()
    var assets_transactions_bulk = self.AssetsTransactions.collection.initializeOrderedBulkOp()
    var assets_utxos_bulk = self.AssetsUtxos.collection.initializeOrderedBulkOp()
    var raw_transaction_bulk = self.RawTransactions.collection.initializeOrderedBulkOp()

    // logger.debug('reverting '+block_data.tx.length+' txs.')
    var txids = []
    var colored_txids = []
    async.eachSeries(block_data.tx.reverse(), function (txid, cb) {
      txids.push(txid)
      self.revert_tx(txid, utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, raw_transaction_bulk, function (err, colored) {
        if (err) return cb(err)
        if (colored) {
          colored_txids.push(txid)
        }
        cb()
      })
    },
    function (err) {
      if (err) return callback(err)
      // logger.debug('executing bulks')
      execute_bulks([utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, raw_transaction_bulk], function (err) {
        if (err) return callback(err)
        txids.forEach(function (txid) {
          self.emit('revertedtransaction', {txid: txid})
        })
        colored_txids.forEach(function (txid) {
          self.emit('revertedcctransaction', {txid: txid})
        })
        self.fix_mempool(function (err) {
          // logger.debug('deleting block')
          self.Blocks.remove(conditions).exec(function (err) {
            if (err) return callback(err)
            // logger.debug('setting hashes')
            self.emit('revertedblock', block_id)
            self.set_last_hash(block_data.previousblockhash)
            self.set_last_block(block_data.height - 1)
            self.set_next_hash(null)
            // logger.debug('done reverting')
          })
        })
      })
    })
  })
}

Scanner.prototype.fix_mempool = function (callback) {
  RawTransactions.update({blockheight: -1}, {iosparsed: false, ccparsed: false}, callback)
}

Scanner.prototype.revert_tx = function (txid, utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, raw_transaction_bulk, callback) {
  var self = this
  var conditions = {
    txid: txid
  }
  console.log('reverting tx ' + txid)
  this.RawTransactions.findOne(conditions).exec(function (err, tx) {
    if (err) return callback(err)
    if (!tx) return callback()
    async.waterfall([
      function (cb) {
        // logger.debug('reverting vin')
        self.revert_vin(tx, utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, cb)<commit_msg>small bug fixed<commit_after>var async = require('async')
var util = require('util')
var events = require('events')
var _ = require('lodash')
var CCTransaction = require('cc-transaction')
var bitcoin = require('bitcoin-async')
var get_assets_outputs = require('cc-get-assets-outputs')

var properties
var bitcoin_rpc
var debug

function Scanner (settings, db) {
  // console.log('Mongoose', db)
  var self = this

  debug = settings.debug
  self.to_revert = []
  self.priority_parse_list = []
  properties = settings.properties
  self.next_hash = settings.next_hash
  self.last_hash = settings.last_hash
  self.last_block = settings.last_block
  self.last_fully_parsed_block = settings.last_fully_parsed_block
  bitcoin_rpc = new bitcoin.Client(settings.rpc_settings)

  self.Blocks = db.model('blocks', require(__dirname + '/models/blocks')(db, properties))
  self.RawTransactions = db.model('rawtransactions', require(__dirname + '/models/rawtransactions')(db, properties))
  self.Utxo = db.model('utxo', require(__dirname + '/models/utxo')(db))
  self.AddressesTransactions = db.model('addressestransactions', require(__dirname + '/models/addressestransactions')(db))
  self.AddressesUtxos = db.model('addressesutxos', require(__dirname + '/models/addressesutxos')(db))
  self.AssetsTransactions = db.model('assetstransactions', require(__dirname + '/models/assetstransactions')(db))
  self.AssetsUtxos = db.model('assetsutxos', require(__dirname + '/models/assetsutxos')(db))
  self.AssetsAddresses = db.model('assetsaddresses', require(__dirname + '/models/assetsaddresses')(db))

  if (process.env.ROLE === properties.roles.SCANNER) {
    self.on('newblock', function (newblock) {
      process.send({to: properties.roles.API, newblock: newblock})
    })
    self.on('newtransaction', function (newtransaction) {
      process.send({to: properties.roles.API, newtransaction: newtransaction})
    })
    self.on('newcctransaction', function (newcctransaction) {
      process.send({to: properties.roles.API, newcctransaction: newcctransaction})
    })
    self.on('revertedblock', function (revertedblock) {
      process.send({to: properties.roles.API, revertedblock: revertedblock})
    })
    self.on('revertedtransaction', function (revertedtransaction) {
      process.send({to: properties.roles.API, revertedtransaction: revertedtransaction})
    })
    self.on('revertedcctransaction', function (revertedcctransaction) {
      process.send({to: properties.roles.API, revertedcctransaction: revertedcctransaction})
    })
    self.on('mempool', function () {
      process.send({to: properties.roles.API, mempool: true})
    })
  }

  self.mempool_cargo = async.cargo(function (tasks, callback) {
    self.parse_mempool_cargo(tasks, callback)
  }, 500)
}

util.inherits(Scanner, events.EventEmitter)

Scanner.prototype.scan_blocks = function (err) {
  var self = this
  if (err) {
    console.error(err)
    return self.scan_blocks()
  }
  var job
  var next_block
  var last_hash

  async.waterfall([
    function (cb) {
      self.get_next_new_block(cb)
    },
    function (l_next_block, l_last_hash, cb) {
      last_hash = l_last_hash
      next_block = l_next_block || 0
      self.get_raw_block(next_block, cb)
    },
    function (raw_block_data, cb) {
      if (!cb) {
        cb = raw_block_data
        raw_block_data = null
      }
      if (!raw_block_data || (raw_block_data.height === next_block - 1 && raw_block_data.hash === last_hash)) {
        setTimeout(function () {
          if (debug) {
            job = 'mempool_scan'
            console.time(job)
          }
          self.parse_new_mempool(cb)
        }, 500)
      } else if (!raw_block_data.previousblockhash || raw_block_data.previousblockhash === last_hash) {
        // logger.debug('parsing block')
        if (debug) {
          job = 'parse_new_block'
          console.time(job)
        }
        self.parse_new_block(raw_block_data, cb)
      } else {
        if (debug) {
          job = 'reverting_block'
          console.time(job)
        }
        self.revert_block(next_block - 1, cb)
      }
    }
  ], function (err) {
    if (debug && job) console.timeEnd(job)
    if (err) self.to_revert = []
    self.scan_blocks(err)
  })
}

Scanner.prototype.revert_block = function (block_height, callback) {
  var self = this
  console.log('Reverting block: ' + block_height)
  var conditions = {
    height: block_height
  }
  self.Blocks.findOne(conditions).exec(function (err, block_data) {
    if (err) return callback(err)
    if (!block_data || !block_data.tx) return callback()
    var block_id = {
      height: block_data.height,
      hash: block_data.hash
    }
    var utxo_bulk = self.Utxo.collection.initializeOrderedBulkOp()
    // var holders_bulk = Holders.collection.initializeOrderedBulkOp()
    var addresses_transactions_bulk = self.AddressesTransactions.collection.initializeOrderedBulkOp()
    var addresses_utxos_bulk = self.AddressesUtxos.collection.initializeOrderedBulkOp()
    var assets_transactions_bulk = self.AssetsTransactions.collection.initializeOrderedBulkOp()
    var assets_utxos_bulk = self.AssetsUtxos.collection.initializeOrderedBulkOp()
    var raw_transaction_bulk = self.RawTransactions.collection.initializeOrderedBulkOp()

    // logger.debug('reverting '+block_data.tx.length+' txs.')
    var txids = []
    var colored_txids = []
    async.eachSeries(block_data.tx.reverse(), function (txid, cb) {
      txids.push(txid)
      self.revert_tx(txid, utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, raw_transaction_bulk, function (err, colored) {
        if (err) return cb(err)
        if (colored) {
          colored_txids.push(txid)
        }
        cb()
      })
    },
    function (err) {
      if (err) return callback(err)
      // logger.debug('executing bulks')
      execute_bulks([utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, raw_transaction_bulk], function (err) {
        if (err) return callback(err)
        txids.forEach(function (txid) {
          self.emit('revertedtransaction', {txid: txid})
        })
        colored_txids.forEach(function (txid) {
          self.emit('revertedcctransaction', {txid: txid})
        })
        self.fix_mempool(function (err) {
          // logger.debug('deleting block')
          self.Blocks.remove(conditions).exec(function (err) {
            if (err) return callback(err)
            // logger.debug('setting hashes')
            self.emit('revertedblock', block_id)
            self.set_last_hash(block_data.previousblockhash)
            self.set_last_block(block_data.height - 1)
            self.set_next_hash(null)
            // logger.debug('done reverting')
          })
        })
      })
    })
  })
}

Scanner.prototype.fix_mempool = function (callback) {
  this.RawTransactions.update({blockheight: -1}, {iosparsed: false, ccparsed: false}, callback)
}

Scanner.prototype.revert_tx = function (txid, utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, raw_transaction_bulk, callback) {
  var self = this
  var conditions = {
    txid: txid
  }
  console.log('reverting tx ' + txid)
  self.RawTransactions.findOne(conditions).exec(function (err, tx) {
    if (err) return callback(err)
    if (!tx) return callback()
    async.waterfall([
      function (cb) {
        // logger.debug('reverting vin')
        self.revert_vin(tx, utxo_bulk, addresses_transactions_bulk, addresses_utxos_bulk, assets_transactions_bulk, assets_utxos_bulk, cb)
{'fraction_non_alphanumeric': 0.12284609840791456, 'fraction_numerical': 0.001441394221319531, 'mean_word_length': 3.5186500888099466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2294999', 'n_tokens_mistral': 5003, 'n_tokens_neox': 4961, 'n_words': 843}