<commit_before>"use strict"
var AHEAD = 1,
	PREFIX = 2,
	INFIX = 4,
	EVENT = 1 << 9;

module.exports = Actions

// built-in plugin
var Plugins = Object.create(null);

Plugins.CORE_ACTIONS = function CORE_ACTIONS(n, self) {
	if (!n.method || n.method == 'FALSE') return false

	if (n.method == 'TRUE') {
		return true
	}

	if (n.method == 'mark') {
		self.pushEvent(self.markEvent(n))
		return true
	}

	// load
	if (self.stage.pos) return false

	if (typeof self.Plugins['LOAD_' + n.method] == 'function')
		return self.Plugins['LOAD_' + n.method](n, self)

	if (typeof Plugins['LOAD_' + n.method] == 'function')
		return Plugins['LOAD_' + n.method](n, self)

	if (typeof self.Plugins[n.method] == 'function')
		return self.Plugins[n.method](n, self)

	if (typeof Plugins[n.method] == 'function')
		return Plugins[n.method](n, self)

	return false
}

Plugins.EOF = function EOF(n, self) {
	if (self.EOF || self.stage.pos != self.src.length) return false
	return self.EOF = true
}

Plugins.LOAD_CRLF = function LOAD_CRLF(n, self) {
	self.plugins.CRLF = Plugins[!n.key && 'ANYCRLF' ||
		n.key == 'CR' && 'CR' ||
		n.key == 'LF' && 'LF' ||
		n.key == 'CRLF' && 'CRLF' || 'ANYCRLF'
	]

	self.stage.line = 1;
	self.stage.col = 1;

	return true

}

Plugins.CRLF_AFTER_CREATE = function CRLF_AFTER_CREATE(n, self) {
	n.loc = Object.create(null)
	n.loc.startLine = 0
	n.loc.startCol = 0
	n.loc.endLine = 0
	n.loc.endCol = 0
}

Plugins.CR = function CR(n, self) {
	if (self.src[self.stage.pos] != '\r') return false
	self.stage.pos++;
	self.stage.line++;
	self.stage.col = 1;
	return true
}

Plugins.LF = function LF(n, self) {
	if (self.src[self.stage.pos] != '\n') return false
	self.stage.pos++;
	self.stage.line++;
	self.stage.col = 1;
	return true
}

Plugins.CRLF = function CRLF(n, self) {
	if (self.src[self.stage.pos] != '\r' ||
		self.src[self.stage.pos + 1] != '\n') return false
	self.stage.pos += 2;
	self.stage.line++;
	self.stage.col = 1;
	return true
}

Plugins.ANYCRLF = function ANYCRLF(n, self) {
	return Plugins.CRLF(n, self) || Plugins.LF(n, self) ||
		Plugins.CR(n, self)
}

Plugins.LOAD_OUTDENT = function LOAD_OUTDENT(n, self) {
	var opt = Object.create(null);
	if (!self.plugins.CRLF) self.plugins.CRLF = Plugins.CRLF

	opt.CRLF = self.plugins.CRLF
	opt.char = n.key == 'SP' && ' ' || '\t';

	self.plugins.CRLF = Plugins.CRLF_OUTDENT.bind(opt)
	self.plugins.OUTDENT = Plugins.OUTDENT.bind(opt.char)
	self.plugins.ON_OUTDENT = Plugins.ON_OUTDENT.bind(opt)

	return true
}

Plugins.OUTDENT = function OUTDENT(n, self) {
	var event = self.markEvent(n, 'OUTDENT', n.extra &&
		(n.extra == 'SP' && 'SP' || 'TAB') ||
		this == ' ' && 'SP' || 'TAB');

	self.branch()
	self.pushEvent(event) // first
	return true
}

Plugins.CRLF_OUTDENT = function CRLF_OUTDENT(n, self) {
	var event;
	// override CRLF
	if (!this.CRLF(n, self)) return false
	event = self.factors[0];
	if (!event || event.key != 'OUTDENT') return false

}

Plugins.ON_OUTDENT = function ON_OUTDENT(n, self) {

}

// helper
function echo(list, a) {
	console.log('...... \n', JSON.stringify(a))
	a = []
	list && list.forEach(function(n, i) {
		console.log('\n', i, '\t', JSON.stringify(n))
	})
}

function non(n) {
	return !!n
}

function copy(via) {
	return Object.assign(Object.create(null), via)
}

function extend(dist, via) {
	if (!via) return dist

	if (Array.isArray(via)) via.forEach(
		function(fn) {
			if (typeof fn == 'function' && fn.name)
				this[fn.name] = fn
		}, dist)
	else
		dist = Object.assign(dist, obj)

	return dist
}

function Actions(inputSourceOrRules, plugins) {
	this.Plugins = extend(Object.create(null), plugins)
	this.plugins = Object.create(null)
	this.plugins.ACTIONS = Plugins.CORE_ACTIONS

	if (typeof inputSourceOrRules == 'string') {
		this.src = inputSourceOrRules
	} else if (inputSourceOrRules) {
		this.rules = inputSourceOrRules
		this.infixes = Object.create(null)
	}
}

Actions.prototype.markEvent = function markEvent(n, key, extra) {
	var event = this.create(n, this.stage.pos, false, this.stage.line, this.stage.col)
	event.type = 'ACTIONS'
	event.method = 'mark'
	if (key) event.key = key
	if (extra) event.extra = extra
	return event
}

Actions.prototype.pushEvent = function pushEvent(event) {
	this.factors.push(event)
	this.stage.flags |= EVENT;
	return event
}

Actions.prototype.init = function init() {
	this.EOF = false
	this.factors = []
	this.stack = []
	this.stage = Object.create(null)
	this.stage.pos = 0;
	this.stage.flags = 0;
}

Actions.prototype.parse = function parse(inputSource) {
	this.src = inputSource
	this.init()
	return this.play(this.rules.defs[this.rules.first])
}

Actions.prototype.retrans = function retrans(rules) {
	if (null == rules) return null
	this.rules = rules
	this.infixes = Object.create(null)
	this.init()
	return this.play(this.rules.defs[this.rules.first])
}

Actions.prototype.play = function play(n) {
	return n && this.repeat(n) && this.build(this.factors) ||
		new Error('Actions failed');
}

Actions.prototype.branch = function branch() {
	this.stack.push(this.factors, copy(this.stage))
	this.factors = []
}

Actions.prototype.revert = function revert() {
	this.stage = this.stack.pop()
	this.factors = this.stack.pop()
}

Actions.prototype.commit = function commit() {
	this.stack.pop()
	this.factors = this.stack.pop()
}

Actions.prototype.build = function build(list) {
	var infix = [];
	if (this.stage.flags & EVENT) {
		this.stage.flags ^= EVENT
		if (!emitEvents(list, this)) return null
	}

	buildFactors(list, infix);
	infix.sort(leftToRight).forEach(buildInfix, list);

	return list.filter(non)
}

function emitEvents(list, ths) {
	return !list.some(function(n, i, list) {
		if (n && n.type == 'ACTIOINS' && n.method == 'mark') {
			if (!this.plugins['ON_' + n.key]) return true
			if (!this.plugins['ON_' + n.key](list, this, i)) return true
			if (list[i]) return true
			return false
		}
	}, ths)
}

function leftToRight(a, b) {
	return a[1] < b[1] && 1 ||
		a[1] > b[1] && -1 ||
		a[0] < b[0] && -1 ||
		a[0] > b[0] && 1 || 0
}

function buildFactors(list, infix) {
	var p, e, i = 0;
	while (i < list.length) {
		p = list[i++]

		if (p.method == 'infix') {
			infix.push([i - 1, list[innerPrecedence(list, i)].precedence])
			continue
		}

		if (!p.type || p.factors) continue

		if (p.method == 'ahead') {
			e = i - 1
			while (e && !list[--e]);
			p.factors = [list[e]]
			list[e] = null
		}

		e = p.end
		while (i < list.length) {
			if (e <= list[i].start) break
			if (!p.factors) p.factors = [];
			p.factors.push(list[i])
			list[i] = null
			i++
		}
	}
}

function buildInfix(a, j) {
	var o, s, j, i = a[0],
		n = this[i];

	while (i && !this[--i]);

	o = this[i] //left
	this[i] = null
	n.start = o.start
<commit_msg>Actions: ahead<commit_after>"use strict"
var AHEAD = 1,
	PREFIX = 2,
	INFIX = 4,
	EVENT = 1 << 9;

module.exports = Actions

// built-in plugin
var Plugins = Object.create(null);

Plugins.CORE_ACTIONS = function CORE_ACTIONS(n, self) {
	if (!n.method || n.method == 'FALSE') return false

	if (n.method == 'TRUE') {
		return true
	}

	if (n.method == 'mark') {
		self.pushEvent(self.markEvent(n))
		return true
	}

	// load
	if (self.stage.pos) return false

	if (typeof self.Plugins['LOAD_' + n.method] == 'function')
		return self.Plugins['LOAD_' + n.method](n, self)

	if (typeof Plugins['LOAD_' + n.method] == 'function')
		return Plugins['LOAD_' + n.method](n, self)

	if (typeof self.Plugins[n.method] == 'function')
		return self.Plugins[n.method](n, self)

	if (typeof Plugins[n.method] == 'function')
		return Plugins[n.method](n, self)

	return false
}

Plugins.EOF = function EOF(n, self) {
	if (self.EOF || self.stage.pos != self.src.length) return false
	return self.EOF = true
}

Plugins.LOAD_CRLF = function LOAD_CRLF(n, self) {
	self.plugins.CRLF = Plugins[!n.key && 'ANYCRLF' ||
		n.key == 'CR' && 'CR' ||
		n.key == 'LF' && 'LF' ||
		n.key == 'CRLF' && 'CRLF' || 'ANYCRLF'
	]

	self.stage.line = 1;
	self.stage.col = 1;

	return true

}

Plugins.CRLF_AFTER_CREATE = function CRLF_AFTER_CREATE(n, self) {
	n.loc = Object.create(null)
	n.loc.startLine = 0
	n.loc.startCol = 0
	n.loc.endLine = 0
	n.loc.endCol = 0
}

Plugins.CR = function CR(n, self) {
	if (self.src[self.stage.pos] != '\r') return false
	self.stage.pos++;
	self.stage.line++;
	self.stage.col = 1;
	return true
}

Plugins.LF = function LF(n, self) {
	if (self.src[self.stage.pos] != '\n') return false
	self.stage.pos++;
	self.stage.line++;
	self.stage.col = 1;
	return true
}

Plugins.CRLF = function CRLF(n, self) {
	if (self.src[self.stage.pos] != '\r' ||
		self.src[self.stage.pos + 1] != '\n') return false
	self.stage.pos += 2;
	self.stage.line++;
	self.stage.col = 1;
	return true
}

Plugins.ANYCRLF = function ANYCRLF(n, self) {
	return Plugins.CRLF(n, self) || Plugins.LF(n, self) ||
		Plugins.CR(n, self)
}

Plugins.LOAD_OUTDENT = function LOAD_OUTDENT(n, self) {
	var opt = Object.create(null);
	if (!self.plugins.CRLF) self.plugins.CRLF = Plugins.CRLF

	opt.CRLF = self.plugins.CRLF
	opt.char = n.key == 'SP' && ' ' || '\t';

	self.plugins.CRLF = Plugins.CRLF_OUTDENT.bind(opt)
	self.plugins.OUTDENT = Plugins.OUTDENT.bind(opt.char)
	self.plugins.ON_OUTDENT = Plugins.ON_OUTDENT.bind(opt)

	return true
}

Plugins.OUTDENT = function OUTDENT(n, self) {
	var event = self.markEvent(n, 'OUTDENT', n.extra &&
		(n.extra == 'SP' && 'SP' || 'TAB') ||
		this == ' ' && 'SP' || 'TAB');

	self.branch()
	self.pushEvent(event) // first
	return true
}

Plugins.CRLF_OUTDENT = function CRLF_OUTDENT(n, self) {
	var event;
	// override CRLF
	if (!this.CRLF(n, self)) return false
	event = self.factors[0];
	if (!event || event.key != 'OUTDENT') return false

}

Plugins.ON_OUTDENT = function ON_OUTDENT(n, self) {

}

// helper
function echo(list, a) {
	console.log('...... \n', JSON.stringify(a))
	a = []
	list && list.forEach(function(n, i) {
		console.log('\n', i, '\t', JSON.stringify(n))
	})
}

function non(n) {
	return !!n
}

function copy(via) {
	return Object.assign(Object.create(null), via)
}

function extend(dist, via) {
	if (!via) return dist

	if (Array.isArray(via)) via.forEach(
		function(fn) {
			if (typeof fn == 'function' && fn.name)
				this[fn.name] = fn
		}, dist)
	else
		dist = Object.assign(dist, obj)

	return dist
}

function Actions(inputSourceOrRules, plugins) {
	this.Plugins = extend(Object.create(null), plugins)
	this.plugins = Object.create(null)
	this.plugins.ACTIONS = Plugins.CORE_ACTIONS

	if (typeof inputSourceOrRules == 'string') {
		this.src = inputSourceOrRules
	} else if (inputSourceOrRules) {
		this.rules = inputSourceOrRules
		this.infixes = Object.create(null)
	}
}

Actions.prototype.markEvent = function markEvent(n, key, extra) {
	var event = this.create(n, this.stage.pos, false, this.stage.line, this.stage.col)
	event.type = 'ACTIONS'
	event.method = 'mark'
	if (key) event.key = key
	if (extra) event.extra = extra
	return event
}

Actions.prototype.pushEvent = function pushEvent(event) {
	this.factors.push(event)
	this.stage.flags |= EVENT;
	return event
}

Actions.prototype.init = function init() {
	this.EOF = false
	this.factors = []
	this.stack = []
	this.stage = Object.create(null)
	this.stage.pos = 0;
	this.stage.flags = 0;
}

Actions.prototype.parse = function parse(inputSource) {
	this.src = inputSource
	this.init()
	return this.play(this.rules.defs[this.rules.first])
}

Actions.prototype.retrans = function retrans(rules) {
	if (null == rules) return null
	this.rules = rules
	this.infixes = Object.create(null)
	this.init()
	return this.play(this.rules.defs[this.rules.first])
}

Actions.prototype.play = function play(n) {
	return n && this.repeat(n) && this.build(this.factors) ||
		new Error('Actions failed');
}

Actions.prototype.branch = function branch() {
	this.stack.push(this.factors, copy(this.stage))
	this.factors = []
}

Actions.prototype.revert = function revert() {
	this.stage = this.stack.pop()
	this.factors = this.stack.pop()
}

Actions.prototype.commit = function commit() {
	this.stack.pop()
	this.factors = this.stack.pop()
}

Actions.prototype.build = function build(list) {
	var infix = [];
	if (this.stage.flags & EVENT) {
		this.stage.flags ^= EVENT
		if (!emitEvents(list, this)) return null
	}

	buildFactors(list, infix);
	infix.sort(leftToRight).forEach(buildInfix, list);

	return list.filter(non)
}

function emitEvents(list, ths) {
	return !list.some(function(n, i, list) {
		if (n && n.type == 'ACTIOINS' && n.method == 'mark') {
			if (!this.plugins['ON_' + n.key]) return true
			if (!this.plugins['ON_' + n.key](list, this, i)) return true
			if (list[i]) return true
			return false
		}
	}, ths)
}

function leftToRight(a, b) {
	return a[1] < b[1] && 1 ||
		a[1] > b[1] && -1 ||
		a[0] < b[0] && -1 ||
		a[0] > b[0] && 1 || 0
}

function buildFactors(list, infix) {
	var p, e, i = 0;
	while (i < list.length) {
		p = list[i++]

		if (p.method == 'infix') {
			infix.push([i - 1, list[innerPrecedence(list, i)].precedence])
			continue
		}

		if (!p.type || p.factors) continue

		if (p.method == 'ahead') {
			e = i - 1
			while (e && !list[--e]);
			p.factors = [list[e]]
			p.start = list[e].start
			list[e] = null
		}

		e = p.end
		while (i < list.length) {
			if (e <= list[i].start) break
			if (!p.factors) p.factors = [];
			p.factors.push(list[i])
			list[i] = null
			i++
		}
	}
}

function buildInfix(a, j) {
	var o, s, j, i = a[0],
		n = this[i];

	while (i && !this[--i]);

	o = this[i] //left
	this[i] = null
	n.start = o.start

{'fraction_non_alphanumeric': 0.18106139438085328, 'fraction_numerical': 0.005351568306823249, 'mean_word_length': 3.582765667574932, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4583082', 'n_tokens_mistral': 5646, 'n_tokens_neox': 5297, 'n_words': 1155}