<commit_before>var libxmljs = require('libxmljs');
var marked = require('marked');

var attributeDefinitionLists = {};

var inlineAttributeLists = [];
var blockAttributeRegex = /(^|\n)(([ \t>]*)(\{[ ]{0,3}:(?:\\\}|[^\}])*\})[ \t]*\r?\n)/g;

function generate(text, enableExtensions, baseURL) {
	if (baseURL) {
		marked.InlineLexer.prototype.outputLink = baseURL;
	}

	if (enableExtensions) {
		var attributeLists = [];
		var match = blockAttributeRegex.exec(text);
		while (match) {
			var index = match.index + match[1].length + match[3].length;
			var lineStart = match.index + match[1].length; //getLineStart(index, text);
			var lineEnd = match.index + match[0].length;
			attributeLists.push({index: index, lineStart: lineStart, match: match});
			text = text.substring(0, lineStart) + text.substring(lineEnd);
			blockAttributeRegex.lastIndex = match.index;
			match = blockAttributeRegex.exec(text);
		}

		var refNameRegex = /\{[ ]{0,3}:([^:]+):([^}]*)/;
		var attributeListContentRegex = /\{[ ]{0,3}:([^}]*)/;
		attributeLists.forEach(function(current) {
			var refNameMatch = refNameRegex.exec(current.match[4]);
			if (refNameMatch) {
				attributeDefinitionLists[refNameMatch[1]] = refNameMatch[2];
			} else {
				var content = attributeListContentRegex.exec(current.match[4]);
				inlineAttributeLists.push({index: current.index, lineStart: current.lineStart, content: content[1].trim(), match: current.match});
			}
		});
	}

	var length = text.length;
	var bounds = {start: 0, contentStart: 0, contentEnd: length, end: length};
	var rootBlock = new Block(bounds, null, null, null, null, null, null, text.substring(bounds.contentStart, bounds.end));
	
	inlineAttributeLists.forEach(function(current) {
		index = current.index;

		if (lineStart) {
			/* not on the first line */
			var previousLineStart = getLineStart(text, current.lineStart - 1); /* backtrack to start of previous line */
			var previousLine = text.substring(previousLineStart, current.lineStart - 1);
			if (!previousLine.indexOf(current.match[3]) && /\S+/.test(previousLine.substring(current.match[3].length))) {
				/* the previous line is not "blank" (excluding indentation characters), move the index back into the previous line's block */
				index = previousLineStart + current.match[3].length;
			}
		}

		var adjacentBlock = findAdjacentBlock2(rootBlock, text, /*current*/ index);
		if (adjacentBlock) {
			adjacentBlock.inlineAttributes = adjacentBlock.inlineAttributes || [];
			adjacentBlock.inlineAttributes.push(current.content);
		}
	});

	var blocks = rootBlock.getBlocks();
	var html = "";
	blocks.forEach(function(current) {
		var tokens = [];
		accumulateTokens(current, tokens);
		tokens.links = {}; // TODO !!!
		html += parser.parse(tokens);
	});
	return html;
}

function accumulateTokens(block, _result) {
	/*
	 * A block has either a set of its own content tokens, or a start/
	 * endToken pair and therefore likely child blocks and tokens.
	 */
	if (block.tokens) {
		block.tokens[0].inlineAttributes = block.inlineAttributes;
		Array.prototype.push.apply(_result, block.tokens);
		return;
	}

	if (!block.startToken || !block.endToken) {
		return; /* should not happen */
	}

	block.startToken.inlineAttributes = block.inlineAttributes;
	_result.push(block.startToken);
	var subBlocks = block.getBlocks();
	subBlocks.forEach(function(current) {
		accumulateTokens(current, _result);
	});
	_result.push(block.endToken);
}

function binarySearch(array, offset, inclusive, low, high) {
	var index;
	if (low === undefined) { low = -1; }
	if (high === undefined) { high = array.length; }
	while (high - low > 1) {
		index = Math.floor((high + low) / 2);
		if (offset <= array[index].start) {
			high = index;
		} else if (inclusive && offset < array[index].end) {
			high = index;
			break;
		} else {
			low = index;
		}
	}
	return high;
}

//function adjustIndex(index, accumulatedDeletions) {
//	for (var i = index; i >= 0; i--) {
//		if (accumulatedDeletions[i]) {
//			return index - accumulatedDeletions[i];
//		}
//	}
//	return index;
//}

function findAdjacentBlock(parentBlock, text, ial) {
	var ialLineStart = ial.lineStart;
	var index = ial.index;
	if (ialLineStart) {
		/* not the first line */
		var previousLineStart = getLineStart(text, ialLineStart - 1);
		var previousLineText = text.substring(previousLineStart, ialLineStart - 1);
		if (previousLineText.indexOf(ial.match[3]) === 0 && previousLineText.length > ial.match[3].length) {
			/* the previous line is not "blank" (excluding indentation characters) */
			index = previousLineStart + ial.match[3].length - 0.5;
		}
	}

	var block = findBlock(parentBlock, index);
	var blocks = block.getBlocks();
	for (var i = 0; i < blocks.length; i++) {
		if (blocks[i].start <= Math.ceil(index) && index < blocks[i].end + 1) {			
			if (blocks[i].start === Math.ceil(index) || Math.ceil(index) === blocks[i].end) {
				return blocks[i];
			} else {
				return findAdjacentBlock(blocks[i], text, ial);
			}
		}
	}
	return null;
}

function findAdjacentBlock2(parentBlock, text, index) {
//	var ialLineStart = ial.lineStart;
//	var index = ial.index;
//	if (ialLineStart) {
//		/* not the first line */
//		var previousLineStart = getLineStart(text, ialLineStart - 1);
//		var previousLineText = text.substring(previousLineStart, ialLineStart - 1);
//		if (previousLineText.indexOf(ial.match[3]) === 0 && previousLineText.length > ial.match[3].length) {
//			/* the previous line is not "blank" (excluding indentation characters) */
//			index = previousLineStart + ial.match[3].length - 0.5;
//		}
//	}

//	var block = findBlock(parentBlock, index);
	var blocks = parentBlock.getBlocks();
	for (var i = 0; i < blocks.length; i++) {
		if (blocks[i].start <= index && index < blocks[i].end) {
			var blockStartLineStart = getLineStart(text, blocks[i].start);
			var indexLineStart = getLineStart(text, index);
			if (blocks[i].start - blockStartLineStart === index - indexLineStart && text.substring(blockStartLineStart, blocks[i].start) === text.substring(indexLineStart, index)) {			
//			if (blocks[i].start === index) {
				return blocks[i];
			} else {
				return findAdjacentBlock2(blocks[i], text, index);
			}
		}
	}
	return null;
}

function findBlock(parentBlock, offset) {
	var blocks = parentBlock.getBlocks();
	if (!blocks.length) {
		return parentBlock;
	}

	var index = binarySearch(blocks, offset, true);
	if (index < blocks.length && blocks[index].start <= offset && offset <= blocks[index].end - 1) {
		return findBlock(blocks[index], offset);
	}
	return parentBlock;
}

/* create the marked parser and its custom renderer */

var markedOptions = {tables: true, gfm: true, headerPrefix: "", xhtml: true};

var customRenderer = new marked.Renderer();
customRenderer.heading = function(text, level, raw) {
	var result = marked.Renderer.prototype.heading.call(this, text, level, raw);<commit_msg>fix IAL look-ahead case and rename findAdjacentBlock2<commit_after>var libxmljs = require('libxmljs');
var marked = require('marked');

var attributeDefinitionLists = {};

var inlineAttributeLists = [];
var blockAttributeRegex = /(^|\n)(([ \t>]*)(\{[ ]{0,3}:(?:\\\}|[^\}])*\})[ \t]*\r?\n)/g;

function generate(text, enableExtensions, baseURL) {
	if (baseURL) {
		marked.InlineLexer.prototype.outputLink = baseURL;
	}

	if (enableExtensions) {
		var attributeLists = [];
		var match = blockAttributeRegex.exec(text);
		while (match) {
			var index = match.index + match[1].length + match[3].length;
			var lineStart = match.index + match[1].length; //getLineStart(index, text);
			var lineEnd = match.index + match[0].length;
			attributeLists.push({index: index, lineStart: lineStart, match: match});
			text = text.substring(0, lineStart) + text.substring(lineEnd);
			blockAttributeRegex.lastIndex = match.index;
			match = blockAttributeRegex.exec(text);
		}

		var refNameRegex = /\{[ ]{0,3}:([^:]+):([^}]*)/;
		var attributeListContentRegex = /\{[ ]{0,3}:([^}]*)/;
		attributeLists.forEach(function(current) {
			var refNameMatch = refNameRegex.exec(current.match[4]);
			if (refNameMatch) {
				attributeDefinitionLists[refNameMatch[1]] = refNameMatch[2];
			} else {
				var content = attributeListContentRegex.exec(current.match[4]);
				inlineAttributeLists.push({index: current.index, lineStart: current.lineStart, content: content[1].trim(), match: current.match});
			}
		});
	}

	var length = text.length;
	var bounds = {start: 0, contentStart: 0, contentEnd: length, end: length};
	var rootBlock = new Block(bounds, null, null, null, null, null, null, text.substring(bounds.contentStart, bounds.end));
	
	inlineAttributeLists.forEach(function(current) {
		index = current.index;

		var checkNextLine = true;
		if (current.lineStart) {
			/* not on the first line */
			var previousLineStart = getLineStart(text, current.lineStart - 1); /* backtrack to start of previous line */
			var previousLine = text.substring(previousLineStart, current.lineStart - 1);
			if (!previousLine.indexOf(current.match[3]) && /\S+/.test(previousLine.substring(current.match[3].length))) {
				/* the previous line is not "blank" (excluding indentation characters), move the index back into the previous line's block */
				checkNextLine = false;
				index = previousLineStart + current.match[3].length;
			}
		}
		if (checkNextLine) {
			/* the "next" line is actually now the current line since all IAL lines have been removed from text */
			var nextLineStart = current.lineStart;
			var nextLineEnd = getLineEnd(text, nextLineStart);
			var nextLine = text.substring(nextLineStart, nextLineEnd - 1);
			if (nextLine.indexOf(current.match[3]) || !/\S+/.test(nextLine.substring(current.match[3].length))) {
				/* the "next" line is "blank" (excluding indentation characters), so this IAL should not be applied anywhere */
				index = -1;
			}
		}

		if (index !== -1) {
			var adjacentBlock = findAdjacentBlock(rootBlock, text, index);
			if (adjacentBlock) {
				adjacentBlock.inlineAttributes = adjacentBlock.inlineAttributes || [];
				adjacentBlock.inlineAttributes.push(current.content);
			}
		}
	});

	var blocks = rootBlock.getBlocks();
	var html = "";
	blocks.forEach(function(current) {
		var tokens = [];
		accumulateTokens(current, tokens);
		tokens.links = {}; // TODO !!!
		html += parser.parse(tokens);
	});
	return html;
}

function accumulateTokens(block, _result) {
	/*
	 * A block has either a set of its own content tokens, or a start/
	 * endToken pair and therefore likely child blocks and tokens.
	 */
	if (block.tokens) {
		block.tokens[0].inlineAttributes = block.inlineAttributes;
		Array.prototype.push.apply(_result, block.tokens);
		return;
	}

	if (!block.startToken || !block.endToken) {
		return; /* should not happen */
	}

	block.startToken.inlineAttributes = block.inlineAttributes;
	_result.push(block.startToken);
	var subBlocks = block.getBlocks();
	subBlocks.forEach(function(current) {
		accumulateTokens(current, _result);
	});
	_result.push(block.endToken);
}

function binarySearch(array, offset, inclusive, low, high) {
	var index;
	if (low === undefined) { low = -1; }
	if (high === undefined) { high = array.length; }
	while (high - low > 1) {
		index = Math.floor((high + low) / 2);
		if (offset <= array[index].start) {
			high = index;
		} else if (inclusive && offset < array[index].end) {
			high = index;
			break;
		} else {
			low = index;
		}
	}
	return high;
}

function findAdjacentBlock(parentBlock, text, index) {
	var blocks = parentBlock.getBlocks();
	for (var i = 0; i < blocks.length; i++) {
		if (blocks[i].start <= index && index < blocks[i].end) {
			var blockStartLineStart = getLineStart(text, blocks[i].start);
			var indexLineStart = getLineStart(text, index);
			if (blocks[i].start - blockStartLineStart === index - indexLineStart && text.substring(blockStartLineStart, blocks[i].start) === text.substring(indexLineStart, index)) {			
//			if (blocks[i].start === index) {
				return blocks[i];
			} else {
				return findAdjacentBlock(blocks[i], text, index);
			}
		}
	}
	return null;
}

function findBlock(parentBlock, offset) {
	var blocks = parentBlock.getBlocks();
	if (!blocks.length) {
		return parentBlock;
	}

	var index = binarySearch(blocks, offset, true);
	if (index < blocks.length && blocks[index].start <= offset && offset <= blocks[index].end - 1) {
		return findBlock(blocks[index], offset);
	}
	return parentBlock;
}

/* create the marked parser and its custom renderer */

var markedOptions = {tables: true, gfm: true, headerPrefix: "", xhtml: true};

var customRenderer = new marked.Renderer();
customRenderer.heading = function(text, level, raw) {
	var result = marked.Renderer.prototype.heading.call(this, text, level, raw);
{'fraction_non_alphanumeric': 0.15512487197668007, 'fraction_numerical': 0.006775388009138896, 'mean_word_length': 4.145520875557357, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1950590', 'n_tokens_mistral': 4436, 'n_tokens_neox': 4060, 'n_words': 986}