<commit_before>var INHERIT = require('inherit'),
    base = require('../tech.js'),
    path = require('path'),
    util = require('util'),
    fs = require('fs');

Tech = exports.Tech = INHERIT(base.Tech, {
    File: INHERIT(base.File, {

        __constructor: function(tech, path, type, parent) {
            this.__base(tech, path, type, parent);
        },

        parseInclude: function(content) {
            var allIncRe = new RegExp(
                [
                    ['/\\*!?', '\\*/'],
                    ['[\'"]', '[\'"]']
                ]
                    .map(function(i) {
                    return ['(?:', i[0], '\\s*borschik:include:(.*?)\\s*', i[1], ')'].join('')
                })
                    .join('|'),
                'g'),
                uniqStr = '\00borschik\00',
                _this = this;

            content = '' + content;

            var incs = [];
            texts = content
                .replace(allIncRe, function(_, incCommFile, incStrFile) {
                var incFile = incCommFile || incStrFile,
                    resultFile = path.join(path.dirname(_this.path), incFile);
                incs.push({
                    file: resultFile,
                    type: incStrFile ? 'string' : 'comment' });
                return uniqStr
            })
                .split(uniqStr)

            // zip texts and includes
            var res = [], t, i;
            while((t = texts.shift()) != null) {
                t && res.push(t);
                (i = incs.shift()) && res.push(i);
            }

            return res
        },

        processInclude: function(baseFile, content) {
            var parsed = content || this.content;

            for(var i = 0; i < parsed.length; i++) {
                var item = parsed[i];

                if(typeof item !== 'string') {
                    if(path.existsSync(item.file)) {
                        var processed =
                            this.processInclude(
                                item.file,
                                this.parseInclude(fs.readFileSync(item.file)));

                        parsed[i] = (item.type === 'comment' ?
                            processed :
                            JSON.stringify(processed));
                    } else {
                        util.error('Base file: ' + baseFile);
                        util.error('Include file: ' + item.file);
                        throw 'File not exists'
                    }
                }
            }

            return parsed.join('');
        },

        processLink: function(path) {
            return this.pathFrom(path);
        }

    })
});<commit_msg>Fix error resolving paths in includes deeper than 2 levels in js tech module<commit_after>var INHERIT = require('inherit'),
    base = require('../tech.js'),
    PATH = require('path'),
    util = require('util'),
    fs = require('fs');

exports.Tech = INHERIT(base.Tech, {

    File: INHERIT(base.File, {

        parseInclude: function(content) {

            var allIncRe = new RegExp([
                        ['/\\*!?', '\\*/'],
                        ['[\'"]', '[\'"]']
                    ]
                    .map(function(i) {
                        return ['(?:', i[0], '\\s*borschik:include:(.*?)\\s*', i[1], ')'].join('');
                    })
                    .join('|'), 'g'),

                uniqStr = '\00borschik\00',
                _this = this;

            content = '' + content;

            var includes = [],
                texts = content
                    .replace(allIncRe, function(_, incCommFile, incStrFile) {

                        var incFile = incCommFile || incStrFile;

                        includes.push({
                            file: _this.pathTo(incFile),
                            type: incStrFile? 'string' : 'comment'
                        });

                        return uniqStr;

                    })
                    .split(uniqStr);

            // zip texts and includes
            var res = [], t, i;
            while((t = texts.shift()) != null) {
                t && res.push(t);
                (i = includes.shift()) && res.push(i);
            }

            return res;

        },

        processInclude: function(baseFile, content) {
            var parsed = content || this.content;

            for(var i = 0; i < parsed.length; i++) {
                var item = parsed[i];

                if(typeof item === 'string') continue;

                if(!PATH.existsSync(item.file)) {
                    throw new Error('File ' + item.file + ' does not exists, base file is ' + baseFile);
                }

                var processed = this.child('include', item.file).process(baseFile);
                parsed[i] = (item.type === 'comment'? processed : JSON.stringify(processed));
            }

            return parsed.join('');
        }

    })
});

{'fraction_non_alphanumeric': 0.14534061047099253, 'fraction_numerical': 0.0030321406913280777, 'mean_word_length': 1.1236051502145923, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3342320', 'n_tokens_mistral': 1360, 'n_tokens_neox': 1249, 'n_words': 266}