<commit_before>
/**
 * @fileoverview This file contains the main implementation of the Spectrum JavaScript
 *               template engine.
 *
 * @author <NAME>
 * @author <NAME>
 */

// var http = require('swipe/httpclient');
var promise = require('promised-io/promise');

var sys = require('sys');

// this gets its own scope to stop it interfering with potential optimisation of code
// in the main parser scope. It's also to avoid having too much stuff in the scope of the
// compiled code (hence the cryptic names)
function _spectrumCompileTemplate (_spectrumTemplateCode) {
    var code;
    try {
        code = eval(_spectrumTemplateCode);
    }
    catch (e) {
        var sys = require('sys');
        sys.debug('TODO handle this compilation error better: ' + sys.inspect(e));
        throw e;
    }
    return code;
}
    
var ast = {};

// taken from https://developer.mozilla.org/En/Core_JavaScript_1.5_Reference/Objects/Array/Map
function map (fun) {
    var len = this.length >>> 0;
    if (typeof fun != "function") {
        throw new TypeError();
    }
    var res = new Array(len);
    var thisp = arguments[1];
    for (var i = 0; i < len; i++) {
         if (i in this) {
              res[i] = fun.call(thisp, this[i], i, this);
         }
    }
    return res;
}


/**
* @private
* @function Make a constructor extend another.
* Makes a subclass by creating a prototype for the child that shares the
* prototype of the parent. Addionally sets the base property of the child
* function to point to the parent function (useful for calling
* `arguments.callee.base.apply(this, arguments)` in the top of the child
* function to allow use of parent constructor).
*
* @param {Function} child
*   Child constructor.
* @param {Function} parent
*   Parent constructor.
*/
function extend (child, parent) {
    var proto = child.prototype,
        p     = function () {};
    p.prototype = parent.prototype;
    child.prototype = new p();
    for (var i in proto) {
        if (proto.hasOwnProperty(i)) {
            child.prototype[i] = proto[i];
        }
    }
    child.base = parent;
}

function quote (str) {
    if (typeof str === 'undefined') return 'undefined';
    // TODO optimise
    str = str.replace(/\"/g, "\\\"");
    str = str.replace(/\r\n|\n|\r/g, '\\n" + \n"');
    return '"' + str + '"';
};

var View = function (proto, args) {
    this._proto = proto;
    this._content = '';
    this.args = args;
};

View.prototype._callNext = function (proto, args) {
    var child = this._proto;
    while (child.base !== proto) {
        child = child.base;
    }
    return child._main.apply(this, args);
};

View.prototype.escape = function (text) {
    if (typeof(text) !== 'string') {
        text = text.toString();
    }
    return text.replace(/[<>&\"]/g, function (char) {
        return '&' + (char === '<' ? 'lt' : char === '>' ? 'gt' : char === '&' ? 'amp' : 'quot') + ';';
    });
};

View.prototype.render = function () {
    var child = this._proto;
    while (child.base !== View) {
        child = child.base;
    }
    return child._main.apply(this, arguments);
};

View.prototype.output = function () {
    for (var i = 0, l = arguments.length; i < l; i++) {
        this._content += arguments[i];
    }
};

View.prototype.content = function () {
    var content = this._content;
    this._content = '';
    return content;
}

var ParseError = function (message) {
    this.message = message;
};

ast.Container = function () {
    this.subnodes = [];
};

ast.Container.prototype.generateContentFunction = function () {
    return 'function(){' + this.renderSubnodes() + '\n}';
};

ast.Container.prototype.renderSubnodes = function () {
    return map.call(this.subnodes, function (node) {
        return node.render();
    }).join('');
};

ast.Container.prototype.lastSubnode = function () {
    return this.subnodes[this.subnodes.length - 1];
};

var Root = ast.Root = function () {
    arguments.callee.base.apply(this, arguments);
    this.methods = [];
    this.methodsByName = {};
    this.parentPath = null;
};

extend(ast.Root, ast.Container);

Root.prototype.generate = function () {
    var generated = '(function(){var viewClass=function(){arguments.callee.base.apply(this,arguments);};'
         + 'function next(view){Array.prototype.shift.call(arguments)._callNext(viewClass,arguments)};'
         + 'viewClass._main=' + this.generateContentFunction() + ';'
         + this.generateMethods()
         + 'return viewClass;})()';
    return generated;
};

Root.prototype.generateMethods = function () {
    return map.call(this.methods, function (method) {
        return method.generate();
    }).join('');
};

Root.prototype.compile = function (parent) {
    var templateClass = _spectrumCompileTemplate(this.generate());
    extend(templateClass, parent);
    return templateClass;
};

Root.prototype.addMethod = function (method) {
    if (method.name in this.methodsByName) {
        throw new ParseError('duplicate method "' + method.name + '"');
    }
    this.methods.push(method);
    this.methodsByName[method.name] = method
};

Root.prototype.setInheritDirective = function (node) {
    if (this.inheritDirective) {
        throw new Error('only one <~inherit ... /> directive allowed per template');
    }
    this.parentPath = node.path;
    this.inheritDirective = node;
};

ast.Method = function (parent) {
    arguments.callee.base.apply(this, arguments);
    this.parent = parent;
    this.preNameWhitespace = '';
    this.name = '';
    this.preArgListWhitespace = '';
    this.argList = '';
    this.postArgListWhitespace = '';
};

extend(ast.Method, ast.Container);

ast.Method.prototype.generate = function () {
    return 'viewClass.prototype.' + this.name + '=function' + (this.argList || '()') + '{' +
           this.renderSubnodes() +
           '\n};';
};

ast.Method.prototype.render = function () {
    return '';
};

ast.Content = function (text) {
    this.text = text;
};

ast.Content.prototype.render = function () {
    return ';this.output(' + quote(this.text) + ');';
};

ast.InheritDirective = function () {
    this.prePathWhitespace = '';
    this.path = '';
    this.postPathWhitespace = '';
};

ast.InheritDirective.prototype.render = function () {
    return '';
};

ast.ExpressionTag = function (code) {
    this.code = code;
};

ast.ExpressionTag.prototype.render = function () {
    return 'this.output(this.escape(' + this.code + '));';
};

ast.CodeBlock = function (code) {
    this.code = code;
};

ast.CodeBlock.prototype.render = function () {
    return this.code;
};

ast.CodeLines = function (code) {
    this.code = code;
};    

ast.CodeLines.prototype.render = function () {
    return this.code;
};

// contexts
var i = 0,
    topLevelContext         = i++,
    endOfInputContext       = i++,
    expressionTagContext    = i++,
    codeBlockContext        = i++,
    codeLinesContext        = i++,
    methodContext           = i++,
    inheritDirectiveContext = i++,
    initBlockContext        = i++,
    beginBlockContext       = i++;
delete i;

// rules - these are converted into RegExps by removing comments and whitespace, escaping forward slashes,
// adding \h for horizontal whitespace ([ \t]) and adding the "g" modifier - see the Makefile
// for example rule{ hello \h* }x is converted to /hello[ \t]*/g
var topLevelRule = rule{
        ([\S\s]*?)   // content
        (?:          // any of...
            (<)%(=)?      // the start of an expression tag or code tag
        |   (\n|^)\h*(:)  // the beginning of a code line
        |   <~(\w+)       // the beginning of a block tag
        |   </~(\w+)>     // the end of a block tag
        |   $             // end of the string
        )
    }x,
    expressionTagRule = rule{
        ([\S\s]*?)        // the expression
        %>                // the closing tag
    }x,
    codeBlockRule = rule{
        ([\S\s]*?)        // the code
        %>                // the closing tag            
    }x,
    codeLineRule = rule{
        (.*?(?:\n|$))     // the rest of the line (including trailing newline)
        (\h*:)?           // possibly the start of another code line
    }x,
    methodRule = rule{
        (\s+)(\w+)        // pre whitespace and name
        (\s*)             // initial whitespace
        (?:
            (\([^\)]*?\))     // bracketed argument list
        )?                    // ...possibly
        (\s*)>            // maybe some whitespace and the end of the opening <~method> tag
    }x,
    inheritDirectiveRule = rule{
        (\s+)             // pre whitespace
        (?:
            none
        |   \"(\S+?)\"       // what we inherit from
        )
        (\s*)/>          // posible whitespace and end of unary tag
    }x,
    initBlockRule = rule{
        >(.*?)</~init>   // content of the init block
    }x,
    beginBlockRule = rule{
        >(.*?)</~begin>  // content of begin block
    }x;

var Template = function (templateRoot, path) {
    this._templateRoot = templateRoot;
    this._path = path;
};

Template.prototype.setAst = function (ast) {
    this._ast = ast;
};

Template.prototype.compile = function (content) {
    var template = this;
    var ast = this.parse(content);
    return this.getParentTemplate(ast).then(function (parent) {
        template.templateClass = ast.compile(parent);
        return template;
    });
};

Template.prototype.render = function (args) {
    var view = this.createInstance(args);
    view.render();
    return view.content();
};


Template.prototype.getParentTemplate = function (ast) {
    var done = new promise.Promise(),
        path = ast.parentPath;
    if (! path && this._path !== '/base.spv') {
        path = '/base.spv';
    }
    if (path) {
        this._templateRoot.loadTemplate(path).then(function (template) {
            done.resolve(template.templateClass);
        }, function (err) {
            done.reject(err);
        });
    }
    else {
        done.resolve(View);
    }
    return done;
};

Template.prototype.createInstance = function (args) {
    return new this.templateClass(this.templateClass, args);
};

Template.prototype.parse = function (content) {
    var root = new ast.Root(),
        context  = topLevelContext,
        position,
        newPosition,
        res,
        tokenStart = 0,
        stack = [root],
        tagStack = [],
        node;

    try {
        SUCCESS: while (true) {
            position = newPosition || 0;
            switch (context) {

                case topLevelContext:
                    topLevelRule.lastIndex = position;
                    res = topLevelRule.exec(content);
                    if (res === null) {
                        throw new ParseError('could not match template');
                    }
                    newPosition = topLevelRule.lastIndex;

                    if (res[1].length > 0 || res[4] && res[4].length > 0) {
                        stack[stack.length - 1].subnodes.push(new ast.Content(res[1] + (res[4] ? res[4] : '')));
                    }
                    if (res[2] && res[3]) { // expression tag start
                        tokenStart = newPosition - 3;
                        context = expressionTagContext;
                    }
                    else if (res[2]) { // code block start
                        tokenStart = newPosition - 2;
                        context = codeBlockContext;
                    }
                    else if (res[5]) { // code line start
                        tokenStart = newPosition - 1;
                        context = codeLinesContext;                  
                    }
                    else if (res[6]) { // block tag start
                        tokenStart = newPosition - (res[6].length + 2);
                        switch (res[6]) {
                            case 'method':
                                context = methodContext;
                                break;
                            case 'inherit':
                                context = inheritDirectiveContext;
                                break;
                            case 'init':
                                context = initBlockContext;
                                break;
                            case 'begin':
                                context = beginBlockContext;
                                break;
                            default:
                                throw new ParseError('unknown block tag <~' + res[6] + '...');
                        }
                    }
                    else if (res[7]) { // block tag end
                        tokenStart = newPosition - (res[7].length + 3);
                        if (tagStack.length === 0 || tagStack[tagStack.length - 1] !== res[7]) {
                            throw new ParseError('closing </~' + res[7] + '> without corresponding opening tag');
                        }
                        stack.pop();
                        tagStack.pop();
                    }
                    else {
                        context = endOfInputContext;
                    }

                    break;

                case expressionTagContext:
                    // TODO escaping of value resulting from expression
                    expressionTagRule.lastIndex = position;
                    res = expressionTagRule.exec(content);
                    if (res === null) {
                        throw new ParseError('cannot find end of expression tag "=>"');
                    }
                    newPosition = expressionTagRule.lastIndex;
                    if (res[1].search(/\S/) === -1) {
                        throw new ParseError("empty expression tag");
                    }
                    stack[stack.length - 1].subnodes.push(new ast.ExpressionTag(res[1]));
                    // TODO return to suitable context for current container (maybe not needed if topLevelContext can handle being in a container)
                    context = topLevelContext;
                    break;

                case codeBlockContext:
                    codeBlockRule.lastIndex = position;
                    res = codeBlockRule.exec(content);
                    if (res === null) {
                        throw new ParseError('cannot find end of code tag "</~js>"');
                    }
                    newPosition = codeBlockRule.lastIndex;

                    var lastSubnode = stack[stack.length - 1].lastSubnode();
                    if (lastSubnode && (lastSubnode instanceof ast.CodeBlock)) {
                        lastSubnode.code += res[1];
                    }
                    else {
                        stack[stack.length - 1].subnodes.push(new ast.CodeBlock(res[1]));
                    }
                    context = topLevelContext;
                    break;

                case codeLinesContext:
                    codeLineRule.lastIndex = position;
                    res = codeLineRule.exec(content);
                    if (res === null) {
                        throw new ParseError('cannot parse code line');
                    }
                    newPosition = codeLineRule.lastIndex;
                    
                    var lastSubnode = stack[stack.length - 1].lastSubnode();
                    if (lastSubnode && (lastSubnode instanceof ast.CodeLines)) {
                        lastSubnode.code += res[1];
                    }
                    else {
                        stack[stack.length - 1].subnodes.push(new ast.CodeLines(res[1]));
                    }

                    if (res[2]) { // if we have the start of a continuing code line
                        tokenStart--;
                        break;
                    }

                    context = topLevelContext;
                    break;

                case methodContext:
                    methodRule.lastIndex = position;
                    res = methodRule.exec(content);
                    if (res === null) {
                        throw new ParseError('malformed/unfinished <~method ... > start tag');
                    }
                    newPosition = methodRule.lastIndex;

                    node = new ast.Method(stack[stack.length - 1]);
                    node.preNameWhitespace     = res[1];
                    node.name                  = res[2];
                    node.preArgListWhitespace  = res[3];
                    node.argList               = res[4];
                    node.postArgListWhitespace = res[5];
    
                    root.addMethod(node);
                    stack[stack.length - 1].subnodes.push(node);
                    tagStack.push('method');
                    stack.push(node);

                    context = topLevelContext;

                    break;

                case inheritDirectiveContext:
                    inheritDirectiveRule.lastIndex = position;
                    res = inheritDirectiveRule.exec(content);
                    if (res === null) {
                        throw new ParseError('malformed/unfinished <~inherit ... /> directive');
                    }
                    newPosition = inheritDirectiveRule.lastIndex;

                    node = new ast.InheritDirective(stack[stack.length - 1]);
                    node.prePathWhitespace = res[1];
                    node.path = res[2];
                    node.postPathWhitespace = res[3];
                    root.setInheritDirective(node);
                    stack[stack.length - 1].subnodes.push(node);

                    context = topLevelContext;

                    break;
                
                case endOfInputContext:
                    while (stack.length > 1) {
                        node = stack.pop;
                        // if we have implicitly closed tags, add exceptions to this exception for them here
                        throw new ParseError('unclosed method tag');
                    }    
                    if (! stack.shift()) {
                        throw new Error('no template on container stack');
                    }
                    break SUCCESS;

                default:
                    throw new Error('unknown parse context: \'' + context + '\'');
<commit_msg>progress on init blocks<commit_after>
/**
 * @fileoverview This file contains the main implementation of the Spectrum JavaScript
 *               template engine.
 *
 * @author <NAME>
 * @author <NAME>
 */

// var http = require('swipe/httpclient');
var promise = require('promised-io/promise');

var sys = require('sys');

// this gets its own scope to stop it interfering with potential optimisation of code
// in the main parser scope. It's also to avoid having too much stuff in the scope of the
// compiled code (hence the cryptic names)
function _spectrumCompileTemplate (_spectrumTemplateCode) {
    var code;
    try {
        code = eval(_spectrumTemplateCode);
    }
    catch (e) {
        var sys = require('sys');
        sys.debug('TODO handle this compilation error better: ' + sys.inspect(e));
        throw e;
    }
    return code;
}
    
var ast = {};

// taken from https://developer.mozilla.org/En/Core_JavaScript_1.5_Reference/Objects/Array/Map
function map (fun) {
    var len = this.length >>> 0;
    if (typeof fun != "function") {
        throw new TypeError();
    }
    var res = new Array(len);
    var thisp = arguments[1];
    for (var i = 0; i < len; i++) {
         if (i in this) {
              res[i] = fun.call(thisp, this[i], i, this);
         }
    }
    return res;
}


/**
* @private
* @function Make a constructor extend another.
* Makes a subclass by creating a prototype for the child that shares the
* prototype of the parent. Addionally sets the base property of the child
* function to point to the parent function (useful for calling
* `arguments.callee.base.apply(this, arguments)` in the top of the child
* function to allow use of parent constructor).
*
* @param {Function} child
*   Child constructor.
* @param {Function} parent
*   Parent constructor.
*/
function extend (child, parent) {
    var proto = child.prototype,
        p     = function () {};
    p.prototype = parent.prototype;
    child.prototype = new p();
    for (var i in proto) {
        if (proto.hasOwnProperty(i)) {
            child.prototype[i] = proto[i];
        }
    }
    child.base = parent;
}

function quote (str) {
    if (typeof str === 'undefined') return 'undefined';
    // TODO optimise
    str = str.replace(/\"/g, "\\\"");
    str = str.replace(/\r\n|\n|\r/g, '\\n" + \n"');
    return '"' + str + '"';
};

var View = function (proto, args) {
    this._proto = proto;
    this._content = '';
    this.args = args;
};

View.prototype._callNext = function (proto, args) {
    var child = this._proto;
    while (child.base !== proto) {
        child = child.base;
    }
    return child._main.apply(this, args);
};

View.prototype.escape = function (text) {
    if (typeof(text) !== 'string') {
        text = text.toString();
    }
    return text.replace(/[<>&\"]/g, function (char) {
        return '&' + (char === '<' ? 'lt' : char === '>' ? 'gt' : char === '&' ? 'amp' : 'quot') + ';';
    });
};

View.prototype.render = function () {
    var child = this._proto;
    while (child.base !== View) {
        child = child.base;
    }
    return child._main.apply(this, arguments);
};

View.prototype.output = function () {
    for (var i = 0, l = arguments.length; i < l; i++) {
        this._content += arguments[i];
    }
};

View.prototype.content = function () {
    var content = this._content;
    this._content = '';
    return content;
}

var ParseError = function (message) {
    this.message = message;
};

ast.Container = function () {
    this.subnodes = [];
};

ast.Container.prototype.generateContentFunction = function () {
    return 'function(){' + this.renderSubnodes() + '\n}';
};

ast.Container.prototype.renderSubnodes = function () {
    return map.call(this.subnodes, function (node) {
        return node.render();
    }).join('');
};

ast.Container.prototype.lastSubnode = function () {
    return this.subnodes[this.subnodes.length - 1];
};

var Root = ast.Root = function () {
    arguments.callee.base.apply(this, arguments);
    this.methods = [];
    this.methodsByName = {};
    this.parentPath = null;
};

extend(ast.Root, ast.Container);

Root.prototype.generate = function () {
    var generated = '(function(){var viewClass=function(){arguments.callee.base.apply(this,arguments);};'
         + 'function next(view){Array.prototype.shift.call(arguments)._callNext(viewClass,arguments)};'
         + 'viewClass._main=' + this.generateContentFunction() + ';'
         + this.generateMethods()
         + 'return viewClass;})()';
    return generated;
};

Root.prototype.generateMethods = function () {
    return map.call(this.methods, function (method) {
        return method.generate();
    }).join('');
};

Root.prototype.compile = function (parent) {
    var templateClass = _spectrumCompileTemplate(this.generate());
    extend(templateClass, parent);
    return templateClass;
};

Root.prototype.addMethod = function (method) {
    if (method.name in this.methodsByName) {
        throw new ParseError('duplicate method "' + method.name + '"');
    }
    this.methods.push(method);
    this.methodsByName[method.name] = method
};

Root.prototype.setInheritDirective = function (node) {
    if (this.inheritDirective) {
        throw new Error('only one <~inherit ... /> directive allowed per template');
    }
    this.parentPath = node.path;
    this.inheritDirective = node;
};

ast.Method = function (parent) {
    arguments.callee.base.apply(this, arguments);
    this.parent = parent;
    this.preNameWhitespace = '';
    this.name = '';
    this.preArgListWhitespace = '';
    this.argList = '';
    this.postArgListWhitespace = '';
};

extend(ast.Method, ast.Container);

ast.Method.prototype.generate = function () {
    return 'viewClass.prototype.' + this.name + '=function' + (this.argList || '()') + '{' +
           this.renderSubnodes() +
           '\n};';
};

ast.Method.prototype.render = function () {
    return '';
};

ast.Content = function (text) {
    this.text = text;
};

ast.Content.prototype.render = function () {
    return ';this.output(' + quote(this.text) + ');';
};

ast.InheritDirective = function () {
    this.prePathWhitespace = '';
    this.path = '';
    this.postPathWhitespace = '';
};

ast.InheritDirective.prototype.render = function () {
    return '';
};

ast.ExpressionTag = function (code) {
    this.code = code;
};

ast.ExpressionTag.prototype.render = function () {
    return 'this.output(this.escape(' + this.code + '));';
};

ast.CodeBlock = function (code) {
    this.code = code;
};

ast.CodeBlock.prototype.render = function () {
    return this.code;
};

ast.CodeLines = function (code) {
    this.code = code;
};    

ast.CodeLines.prototype.render = function () {
    return this.code;
};

// contexts
var i = 0,
    topLevelContext         = i++,
    endOfInputContext       = i++,
    expressionTagContext    = i++,
    codeBlockContext        = i++,
    codeLinesContext        = i++,
    methodContext           = i++,
    inheritDirectiveContext = i++,
    initBlockContext        = i++,
    beginBlockContext       = i++;
delete i;

// rules - these are converted into RegExps by removing comments and whitespace, escaping forward slashes,
// adding \h for horizontal whitespace ([ \t]) and adding the "g" modifier - see the Makefile
// for example rule{ hello \h* }x is converted to /hello[ \t]*/g
var topLevelRule = rule{
        ([\S\s]*?)   // content
        (?:          // any of...
            (<)%(=)?      // the start of an expression tag or code tag
        |   (\n|^)\h*(:)  // the beginning of a code line
        |   <~(\w+)       // the beginning of a block tag
        |   </~(\w+)>     // the end of a block tag
        |   $             // end of the string
        )
    }x,
    expressionTagRule = rule{
        ([\S\s]*?)        // the expression
        %>                // the closing tag
    }x,
    codeBlockRule = rule{
        ([\S\s]*?)        // the code
        %>                // the closing tag            
    }x,
    codeLineRule = rule{
        (.*?(?:\n|$))     // the rest of the line (including trailing newline)
        (\h*:)?           // possibly the start of another code line
    }x,
    methodRule = rule{
        (\s+)(\w+)        // pre whitespace and name
        (\s*)             // initial whitespace
        (?:
            (\([^\)]*?\))     // bracketed argument list
        )?                    // ...possibly
        (\s*)>            // maybe some whitespace and the end of the opening <~method> tag
    }x,
    inheritDirectiveRule = rule{
        (\s+)             // pre whitespace
        (?:
            none
        |   \"(\S+?)\"       // what we inherit from
        )
        (\s*)/>          // posible whitespace and end of unary tag
    }x,
    initBlockRule = rule{
        >(.*?)</~init>   // content of the init block
    }x,
    beginBlockRule = rule{
        >(.*?)</~begin>  // content of begin block
    }x;

var Template = function (templateRoot, path) {
    this._templateRoot = templateRoot;
    this._path = path;
};

Template.prototype.setAst = function (ast) {
    this._ast = ast;
};

Template.prototype.compile = function (content) {
    var template = this;
    var ast = this.parse(content);
    return this.getParentTemplate(ast).then(function (parent) {
        template.templateClass = ast.compile(parent);
        return template;
    });
};

Template.prototype.render = function (args) {
    var view = this.createInstance(args);
    view.render();
    return view.content();
};


Template.prototype.getParentTemplate = function (ast) {
    var done = new promise.Promise(),
        path = ast.parentPath;
    if (! path && this._path !== '/base.spv') {
        path = '/base.spv';
    }
    if (path) {
        this._templateRoot.loadTemplate(path).then(function (template) {
            done.resolve(template.templateClass);
        }, function (err) {
            done.reject(err);
        });
    }
    else {
        done.resolve(View);
    }
    return done;
};

Template.prototype.createInstance = function (args) {
    return new this.templateClass(this.templateClass, args);
};

Template.prototype.parse = function (content) {
    var root = new ast.Root(),
        context  = topLevelContext,
        position,
        newPosition,
        res,
        tokenStart = 0,
        stack = [root],
        tagStack = [],
        node;

    try {
        SUCCESS: while (true) {
            position = newPosition || 0;
            switch (context) {

                case topLevelContext:
                    topLevelRule.lastIndex = position;
                    res = topLevelRule.exec(content);
                    if (res === null) {
                        throw new ParseError('could not match template');
                    }
                    newPosition = topLevelRule.lastIndex;

                    if (res[1].length > 0 || res[4] && res[4].length > 0) {
                        stack[stack.length - 1].subnodes.push(new ast.Content(res[1] + (res[4] ? res[4] : '')));
                    }
                    if (res[2] && res[3]) { // expression tag start
                        tokenStart = newPosition - 3;
                        context = expressionTagContext;
                    }
                    else if (res[2]) { // code block start
                        tokenStart = newPosition - 2;
                        context = codeBlockContext;
                    }
                    else if (res[5]) { // code line start
                        tokenStart = newPosition - 1;
                        context = codeLinesContext;                  
                    }
                    else if (res[6]) { // block tag start
                        tokenStart = newPosition - (res[6].length + 2);
                        switch (res[6]) {
                            case 'method':
                                context = methodContext;
                                break;
                            case 'inherit':
                                context = inheritDirectiveContext;
                                break;
                            case 'init':
                                context = initBlockContext;
                                break;
                            case 'begin':
                                context = beginBlockContext;
                                break;
                            default:
                                throw new ParseError('unknown block tag <~' + res[6] + '...');
                        }
                    }
                    else if (res[7]) { // block tag end
                        tokenStart = newPosition - (res[7].length + 3);
                        if (tagStack.length === 0 || tagStack[tagStack.length - 1] !== res[7]) {
                            throw new ParseError('closing </~' + res[7] + '> without corresponding opening tag');
                        }
                        stack.pop();
                        tagStack.pop();
                    }
                    else {
                        context = endOfInputContext;
                    }

                    break;

                case expressionTagContext:
                    // TODO escaping of value resulting from expression
                    expressionTagRule.lastIndex = position;
                    res = expressionTagRule.exec(content);
                    if (res === null) {
                        throw new ParseError('cannot find end of expression tag "=>"');
                    }
                    newPosition = expressionTagRule.lastIndex;
                    if (res[1].search(/\S/) === -1) {
                        throw new ParseError("empty expression tag");
                    }
                    stack[stack.length - 1].subnodes.push(new ast.ExpressionTag(res[1]));
                    // TODO return to suitable context for current container (maybe not needed if topLevelContext can handle being in a container)
                    context = topLevelContext;
                    break;

                case codeBlockContext:
                    codeBlockRule.lastIndex = position;
                    res = codeBlockRule.exec(content);
                    if (res === null) {
                        throw new ParseError('cannot find end of code tag "</~js>"');
                    }
                    newPosition = codeBlockRule.lastIndex;

                    var lastSubnode = stack[stack.length - 1].lastSubnode();
                    if (lastSubnode && (lastSubnode instanceof ast.CodeBlock)) {
                        lastSubnode.code += res[1];
                    }
                    else {
                        stack[stack.length - 1].subnodes.push(new ast.CodeBlock(res[1]));
                    }
                    context = topLevelContext;
                    break;

                case codeLinesContext:
                    codeLineRule.lastIndex = position;
                    res = codeLineRule.exec(content);
                    if (res === null) {
                        throw new ParseError('cannot parse code line');
                    }
                    newPosition = codeLineRule.lastIndex;
                    
                    var lastSubnode = stack[stack.length - 1].lastSubnode();
                    if (lastSubnode && (lastSubnode instanceof ast.CodeLines)) {
                        lastSubnode.code += res[1];
                    }
                    else {
                        stack[stack.length - 1].subnodes.push(new ast.CodeLines(res[1]));
                    }

                    if (res[2]) { // if we have the start of a continuing code line
                        tokenStart--;
                        break;
                    }

                    context = topLevelContext;
                    break;

                case methodContext:
                    methodRule.lastIndex = position;
                    res = methodRule.exec(content);
                    if (res === null) {
                        throw new ParseError('malformed/unfinished <~method ... > start tag');
                    }
                    newPosition = methodRule.lastIndex;

                    node = new ast.Method(stack[stack.length - 1]);
                    node.preNameWhitespace     = res[1];
                    node.name                  = res[2];
                    node.preArgListWhitespace  = res[3];
                    node.argList               = res[4];
                    node.postArgListWhitespace = res[5];
    
                    root.addMethod(node);
                    stack[stack.length - 1].subnodes.push(node);
                    tagStack.push('method');
                    stack.push(node);

                    context = topLevelContext;

                    break;

                case inheritDirectiveContext:
                    inheritDirectiveRule.lastIndex = position;
                    res = inheritDirectiveRule.exec(content);
                    if (res === null) {
                        throw new ParseError('malformed/unfinished <~inherit ... /> directive');
                    }
                    newPosition = inheritDirectiveRule.lastIndex;

                    node = new ast.InheritDirective(stack[stack.length - 1]);
                    node.prePathWhitespace = res[1];
                    node.path = res[2];
                    node.postPathWhitespace = res[3];
                    root.setInheritDirective(node);
                    stack[stack.length - 1].subnodes.push(node);

                    context = topLevelContext;

                    break;

                case initBlockContext:
                    initBlockRule.lastIndex = position;
                    res = initBlockRule.exec(content);
                    if (res === null) {
                        throw new ParseError('malformed/unfinished <~init> ... </~init> block')
                    }
                    newPosition = initBlockRul.lastIndex;

                    node = new ast.InitBlock(stack[stack.length - 1]);
                    
                
                case endOfInputContext:
                    while (stack.length > 1) {
                        node = stack.pop;
                        // if we have implicitly closed tags, add exceptions to this exception for them here
                        throw new ParseError('unclosed method tag');
                    }    
                    if (! stack.shift()) {
                        throw new Error('no template on container stack');
                    }
                    break SUCCESS;

                default:
                    throw new Error('unknown parse context: \'' + context + '\'');

{'fraction_non_alphanumeric': 0.12385197190707725, 'fraction_numerical': 0.0034846029173419774, 'mean_word_length': 1.525651521353527, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 57, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2805452', 'n_tokens_mistral': 9999, 'n_tokens_neox': 9491, 'n_words': 2426}