<commit_before>var _ = require("lodash");
var Q = require("q");
var path = require("path");
var nunjucks = require("nunjucks");

var git = require("./utils/git");
var stringUtils = require("./utils/string");
var fs = require("./utils/fs");
var pkg = require("../package.json");


// The loader should handle relative and git url
var BookLoader = nunjucks.Loader.extend({
	async: true,

    init: function(book) {
    	this.book = book;
    },

    getSource: function(fileurl, callback) {
    	var that = this;

		git.resolveFile(fileurl)
		.then(function(filepath) {
			// Is local file
			if (!filepath) filepath = path.resolve(that.book.root, fileurl);
			else that.book.log.debug.ln("resolve from git", fileurl, "to", filepath)

			//  Read file from absolute path
			return fs.readFile(filepath)
    		.then(function(source) {
    			return {
    				src: source.toString(),
    				path: filepath
    			}
    		});
		})
		.nodeify(callback);
    },

    resolve: function(from, to) {
        return path.resolve(path.dirname(from), to);
    }
});


var TemplateEngine = function(book) {
	this.book = book;
    this.log = this.book.log;

	// Nunjucks env
	this.env = new nunjucks.Environment(
		new BookLoader(book),
		{
			// Escaping is done after by the markdown parser
			autoescape: false,

			// Tags
			tags: {
				blockStart: '{%',
				blockEnd: '%}',
				variableStart: '{{',
				variableEnd: '}}',
				commentStart: '{###',
				commentEnd: '###}'
			}
		}
	);

    // List of tags shortcuts
    this.shortcuts = [];

    // Map of blocks
    this.blocks = {};

    // Bind methods
    _.bindAll(this);
};

// Process a block in a context
TemplateEngine.prototype.processBlock = function(blk) {
    if (_.isString(blk)) blk = { body: blk };
    blk = _.defaults(blk, {
        parse: false
    });
    blk.id = _.uniqueId("blk");

    // Add to global map
    this.blocks[blk.id] = blk;

    // If don't parse id, return it as a macro
    if (!blk.parse) return "%+%"+blk.id+"%+%";

    return blk.body;
};

// Replace blocks by body after processing
// This is done to avoid that markdown processer parse the block content
TemplateEngine.prototype.replaceBlocks = function(content) {
    var that = this;

    return content.replace(/\%\+\%([\s\S]+?)\%\+\%/g, function(match, key) {
        var blk = that.blocks[key];
        if (!blk || blk.parse) return match;
        return blk.body;
    });
};

// Bind a function to a context
TemplateEngine.prototype.bindContext = function(func) {
    var that = this;

    return function() {
        var ctx = {
            ctx: this.ctx,
            book: that.book
        };
<commit_msg>Remove used blocks from templates<commit_after>var _ = require("lodash");
var Q = require("q");
var path = require("path");
var nunjucks = require("nunjucks");

var git = require("./utils/git");
var stringUtils = require("./utils/string");
var fs = require("./utils/fs");
var pkg = require("../package.json");


// The loader should handle relative and git url
var BookLoader = nunjucks.Loader.extend({
	async: true,

    init: function(book) {
    	this.book = book;
    },

    getSource: function(fileurl, callback) {
    	var that = this;

		git.resolveFile(fileurl)
		.then(function(filepath) {
			// Is local file
			if (!filepath) filepath = path.resolve(that.book.root, fileurl);
			else that.book.log.debug.ln("resolve from git", fileurl, "to", filepath)

			//  Read file from absolute path
			return fs.readFile(filepath)
    		.then(function(source) {
    			return {
    				src: source.toString(),
    				path: filepath
    			}
    		});
		})
		.nodeify(callback);
    },

    resolve: function(from, to) {
        return path.resolve(path.dirname(from), to);
    }
});


var TemplateEngine = function(book) {
	this.book = book;
    this.log = this.book.log;

	// Nunjucks env
	this.env = new nunjucks.Environment(
		new BookLoader(book),
		{
			// Escaping is done after by the markdown parser
			autoescape: false,

			// Tags
			tags: {
				blockStart: '{%',
				blockEnd: '%}',
				variableStart: '{{',
				variableEnd: '}}',
				commentStart: '{###',
				commentEnd: '###}'
			}
		}
	);

    // List of tags shortcuts
    this.shortcuts = [];

    // Map of blocks
    this.blocks = {};

    // Bind methods
    _.bindAll(this);
};

// Process a block in a context
TemplateEngine.prototype.processBlock = function(blk) {
    if (_.isString(blk)) blk = { body: blk };

    blk = _.defaults(blk, {
        parse: false
    });
    blk.id = _.uniqueId("blk");

    //Parsable block, just return it
    if (blk.parse) {
        return blk.body;
    }

    // Add to global map
    this.blocks[blk.id] = blk;

    // Return it as a macro
    return "%+%"+blk.id+"%+%";
};

// Replace blocks by body after processing
// This is done to avoid that markdown processer parse the block content
TemplateEngine.prototype.replaceBlocks = function(content) {
    var that = this;

    return content.replace(/\%\+\%([\s\S]+?)\%\+\%/g, function(match, key) {
        var blk = that.blocks[key];
        if (!blk) return match;

        var body = blk.body;

        // Remove it from map
        delete that.blocks[key];

        return body;
    });
};

// Bind a function to a context
TemplateEngine.prototype.bindContext = function(func) {
    var that = this;

    return function() {
        var ctx = {
            ctx: this.ctx,
            book: that.book
        };

{'fraction_non_alphanumeric': 0.1541436464088398, 'fraction_numerical': 0.0, 'mean_word_length': 2.474728087012156, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7100571', 'n_tokens_mistral': 1914, 'n_tokens_neox': 1758, 'n_words': 459}