<commit_before>require('classtool');

function ClassSpec(b) {
	var fs = require('fs');
	var Block = require('libcoin/Block');
	var Deserialize = require('libcoin/Deserialize');
	var Parser = require('libcoin/util/BinaryParser');

	function HeaderDB(b) {
		this.network = b.network;
		this.fd = null;
		this.blocks = {};
		this.bestBlock = null;
	};

	HeaderDB.prototype.size = function() {
		return Object.keys(this.blocks).length;
	};

	HeaderDB.prototype.locator = function(block) {
		if (!block)
			block = this.bestBlock;

		var step = 1;
		var loc = [];
		while (block) {
			loc.push(block.calcHash());

			for (var i = 0; block && i < step; i++)
				block = block.prev;
			if (loc.length > 10)
				step *= 2;
		}

		loc.push(this.network.genesisBlock.hash);

		return loc;
	};

	HeaderDB.prototype.add = function(block) {
		var hash = block.calcHash();
		var hashStr = hash.toString();
		var prevHashStr = block.prev_hash.toString();
		var curWork = Deserialize.intFromCompact(block.bits);

		if (hashStr in this.blocks)
			throw new Error("duplicate block");

		var bestChain = false;

		var reorg = {
			oldBest: null,
			conn: 0,
			disconn: 0,
		};

		if (this.size() == 0) {
			if (this.network.genesisBlock.hash.toString() !=
			    hashStr)
				throw new Error("Invalid genesis block");

			block.prev = null;
			block.height = 0;
			block.work = curWork;
			bestChain = true;
		} else {
			var prevBlock = this.blocks[prevHashStr];
			if (!prevBlock)
				throw new Error("orphan block; prev not found");

			block.prev = prevBlock;
			block.height = prevBlock.height + 1;
			block.work = prevBlock.work + curWork;

			if (block.work > this.bestBlock.work)
				bestChain = true;
		}

		// add to by-hash index
		this.blocks[hashStr] = block;

		if (bestChain) {
			var oldBest = this.bestBlock;
			var newBest = block;

			reorg.oldBest = oldBest;

			// likely case: new best chain has greater height
			if (!oldBest) {
				while (newBest) {
					newBest = newBest.prev;
					reorg.conn++;
				}
			} else {
				while (newBest &&
				       (newBest.height > oldBest.height)) {
					newBest = newBest.prev;
					reorg.conn++;
				}
			}

			// unlikely: old best chain has greater height
			while (oldBest && newBest &&
			       (oldBest.height > newBest.height)) {
				oldBest = oldBest.prev;
				reorg.disconn++;
			}

			// height matches, but still walking parallel
			while (oldBest && newBest && (oldBest != newBest)) {
				newBest = newBest.prev;
				reorg.conn++;

				oldBest = oldBest.prev;
				reorg.disconn++;
			}

			// reorg analyzed, updated best-chain pointer
			this.bestBlock = block;
		}

		return reorg;
	};

	HeaderDB.prototype.addBuf = function(buf) {
		var block = new Block();
		var parser = new Parser(buf);
		block.parse(parser, true);
		this.add(block);
	};

	HeaderDB.prototype.readFile = function(filename) {
		var fd = fs.openSync(filename, 'r');
		var stats = fs.fstatSync(fd);
		if (stats.size % 80 != 0)
			throw new Error("Corrupted header db");

		while (1) {
			var buf = new Buffer(80);
			var bread = fs.readSync(fd, buf, 0, 80, null);
			if (bread < 80)
				break;

			this.addBuf(buf);
		}

		fs.closeSync(fd);
	};

	HeaderDB.prototype.writeFile = function(filename) {
		var block = this.bestBlock;
		var data = [];
		while (block) {
			var s = block.getHeader();
			data.push(s);
			block = block.prev;
		}

		data.reverse();

		var fd = fs.openSync(filename, 'w');

		data.foreach(function(datum) {
			fs.writeSync(fd, datum, 0, 80, null);
		});

		fs.closeSync(fd);
	};

	return HeaderDB;
};
module.defineClass(ClassSpec);

<commit_msg>HeaderDB: maintain by-height index. Use in locator generation.<commit_after>require('classtool');

function ClassSpec(b) {
	var assert = require('assert');
	var fs = require('fs');
	var Block = require('libcoin/Block');
	var Deserialize = require('libcoin/Deserialize');
	var Parser = require('libcoin/util/BinaryParser');

	function HeaderDB(b) {
		this.network = b.network;
		this.fd = null;
		this.blocks = {};
		this.byHeight = [];
		this.bestBlock = null;
	};

	HeaderDB.prototype.size = function() {
		return Object.keys(this.blocks).length;
	};

	HeaderDB.prototype.locator = function(block) {
		if (!block)
			block = this.bestBlock;

		var step = 1;
		var start = 0;
		var loc = [];
		for (var i = block.height; i > 0; i -= step, ++start) {
			if (start >= 10)
				step *= 2;
			loc.push(this.byHeight[i].calcHash());
		}
		assert.equal(this.byHeight[0].calcHash().toString(),
			    this.network.genesisBlock.hash.toString());
		loc.push(this.byHeight[0].calcHash());

		return loc;
	};

	HeaderDB.prototype.add = function(block) {
		var hash = block.calcHash();
		var hashStr = hash.toString();
		var prevHashStr = block.prev_hash.toString();
		var curWork = Deserialize.intFromCompact(block.bits);

		if (hashStr in this.blocks)
			throw new Error("duplicate block");

		var bestChain = false;

		var reorg = {
			oldBest: null,
			conn: 0,
			disconn: 0,
		};

		if (this.size() == 0) {
			if (this.network.genesisBlock.hash.toString() !=
			    hashStr)
				throw new Error("Invalid genesis block");

			block.prev = null;
			block.height = 0;
			block.work = curWork;
			bestChain = true;
		} else {
			var prevBlock = this.blocks[prevHashStr];
			if (!prevBlock)
				throw new Error("orphan block; prev not found");

			block.prev = prevBlock;
			block.height = prevBlock.height + 1;
			block.work = prevBlock.work + curWork;

			if (block.work > this.bestBlock.work)
				bestChain = true;
		}

		// add to by-hash index
		this.blocks[hashStr] = block;

		if (bestChain) {
			var oldBest = this.bestBlock;
			var newBest = block;

			reorg.oldBest = oldBest;

			// likely case: new best chain has greater height
			if (!oldBest) {
				while (newBest) {
					newBest = newBest.prev;
					reorg.conn++;
				}
			} else {
				while (newBest &&
				       (newBest.height > oldBest.height)) {
					newBest = newBest.prev;
					reorg.conn++;
				}
			}

			// unlikely: old best chain has greater height
			while (oldBest && newBest &&
			       (oldBest.height > newBest.height)) {
				oldBest = oldBest.prev;
				reorg.disconn++;
			}

			// height matches, but still walking parallel
			while (oldBest && newBest && (oldBest != newBest)) {
				newBest = newBest.prev;
				reorg.conn++;

				oldBest = oldBest.prev;
				reorg.disconn++;
			}

			var shuf = (reorg.conn > reorg.disconn) ?
				   reorg.conn : reorg.disconn;

			// reorg analyzed, updated best-chain pointer
			this.bestBlock = block;

			// update by-height index
			var ptr = block;
			for (var idx = block.height; 
			     idx > (block.height - shuf); idx--) {
				if (idx < 0)
					break;
				this.byHeight[idx] = ptr;
				ptr = ptr.prev;
			}
		}

		return reorg;
	};

	HeaderDB.prototype.addBuf = function(buf) {
		var block = new Block();
		var parser = new Parser(buf);
		block.parse(parser, true);
		this.add(block);
	};

	HeaderDB.prototype.readFile = function(filename) {
		var fd = fs.openSync(filename, 'r');
		var stats = fs.fstatSync(fd);
		if (stats.size % 80 != 0)
			throw new Error("Corrupted header db");

		while (1) {
			var buf = new Buffer(80);
			var bread = fs.readSync(fd, buf, 0, 80, null);
			if (bread < 80)
				break;

			this.addBuf(buf);
		}

		fs.closeSync(fd);
	};

	HeaderDB.prototype.writeFile = function(filename) {
		var block = this.bestBlock;
		var data = [];
		while (block) {
			var s = block.getHeader();
			data.push(s);
			block = block.prev;
		}

		data.reverse();

		var fd = fs.openSync(filename, 'w');

		data.foreach(function(datum) {
			fs.writeSync(fd, datum, 0, 80, null);
		});

		fs.closeSync(fd);
	};

	return HeaderDB;
};
module.defineClass(ClassSpec);


{'fraction_non_alphanumeric': 0.14845173041894352, 'fraction_numerical': 0.0067655477491543065, 'mean_word_length': 2.8784056508577196, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6662346', 'n_tokens_mistral': 3212, 'n_tokens_neox': 2648, 'n_words': 637}