<commit_before>var INHERIT = require('inherit'),
    base = require('../tech.js'),

    PATH = require('path'),
    FS = require('fs'),

    stringRe = "(?:(?:'[^'\\r\\n]*')|(?:\"[^\"\\r\\n]*\"))",
    urlRe = "(?:(?:url\\(\\s*" + stringRe + "\\s*\\))|(?:url\\(\\s*[^\\s\\r\\n'\"]*\\s*\\)))",
    commentRe = '(?:/\\*[^*]*\\*+(?:[^/][^*]*\\*+)*/)',
    importRe = '(?:\\@import\\s+(' + urlRe + '|' + stringRe + '))',
    allRe = new RegExp(commentRe + '|' + importRe + '|' + urlRe, 'g'),
    urlStringRx = new RegExp('^' + urlRe + '$');

Tech = exports.Tech = INHERIT(base.Tech, {
    File: INHERIT(base.File, {

        __constructor: function(tech, path, type, parent) {
            this.__base(tech, path, type, parent);
            this.root = PATH.dirname(path);
        },

        parseInclude: function(content) {
            var m, found = [];

            if (Buffer.isBuffer(content)) content = content.toString('utf8');

            while (m = allRe.exec(content)) {
                if (m[0].lastIndexOf('/*', 0) === 0) {
                    // skip comment
                } else if (m[0].charAt(0) === '@') {
                    // @import
                    var url = parseUrl(m[1]);
                    if (isRelative(url)) found.push({ type: 'import', url: url, range: [ m.index, allRe.lastIndex ] });
                } else if (urlStringRx.test(m[0])) {
                    // url(...)
                    var url = parseUrl(m[0]);
                    if (isRelative(url)) found.push({ type: 'url', url: url, range: [ m.index, allRe.lastIndex-1 ] });
                } else {
                    throw new Error('Failed to match: ' + m[0]);
                }
            }

            return makeParsed(found, content);
        },

        processInclude: function(path, content) {
            var parsed = content || this.content;

            for(var i = 0; i < parsed.length; i++) {
                var item = parsed[i];

                if (typeof item === 'string') {
                    parsed[i] = item;
                    continue;
                }

                var dir = PATH.dirname(path),
                    url = item.url;

                url = PATH.resolve(dir, url);
                if (item.type === 'import') {
                    var file = PATH.relative(this.root, url);
                    parsed[i] = '/* ' + file + ' begin */\n' +
                        this.processInclude(url, this.parseInclude(FS.readFileSync(url))) +
                        '\n/* ' + file + ' end */\n';
                } else if (item.type === 'url') {
                    parsed[i] = 'url(' + PATH.relative(this.root, url) + ')';
                }
            }

            return parsed.join('');
        },

        processLink: function(path) {
            return JSON.stringify(this.pathFrom(path))
        }

    })
});

function parseUrl(url) {
    if (url.lastIndexOf('url(', 0) === 0) url = url.replace(/^url\(\s*/, '').replace(/\s*\)$/, '');

    if (url.charAt(0) === '\'' || url.charAt(0) === '"') url = url.substr(1, url.length - 2);

    return url;
}

function isRelative(url) {
    return !((url.lastIndexOf('/', 0) === 0) || /^\w+:/.test(url));
}<commit_msg>Fix error resolving paths in includes deeper than 2 levels in css-fast tech module<commit_after>var INHERIT = require('inherit'),
    base = require('../tech.js'),

    stringRe = "(?:(?:'[^'\\r\\n]*')|(?:\"[^\"\\r\\n]*\"))",
    urlRe = "(?:(?:url\\(\\s*" + stringRe + "\\s*\\))|(?:url\\(\\s*[^\\s\\r\\n'\"]*\\s*\\)))",
    commentRe = '(?:/\\*[^*]*\\*+(?:[^/][^*]*\\*+)*/)',
    importRe = '(?:\\@import\\s+(' + urlRe + '|' + stringRe + '))',
    allRe = new RegExp(commentRe + '|' + importRe + '|' + urlRe, 'g'),
    urlStringRx = new RegExp('^' + urlRe + '$');

exports.Tech = INHERIT(base.Tech, {

    File: INHERIT(base.File, {

        parseInclude: function(content) {
            var m, found = [];

            if (Buffer.isBuffer(content)) content = content.toString('utf8');

            while (m = allRe.exec(content)) {
                if (m[0].lastIndexOf('/*', 0) === 0) {
                    // skip comment
                } else if (m[0].charAt(0) === '@') {
                    // @import
                    var url = parseUrl(m[1]);
                    if (isRelative(url)) found.push({ type: 'include', url: url, range: [ m.index, allRe.lastIndex ] });
                } else if (urlStringRx.test(m[0])) {
                    // url(...)
                    var url = parseUrl(m[0]);
                    if (isRelative(url)) found.push({ type: 'link', url: url, range: [ m.index, allRe.lastIndex-1 ] });
                } else {
                    throw new Error('Failed to match: ' + m[0]);
                }
            }

            return makeParsed(found, content);
        },

        processInclude: function(path, content) {
            var parsed = content || this.content;

            for(var i = 0; i < parsed.length; i++) {
                var item = parsed[i];

                if (typeof item === 'string') continue;

                if (item.type === 'include') {
                    parsed[i] = '/* ' + item.url + ' begin */\n' +
                        this.child('include', item.url).process(path) +
                        '\n/* ' + item.url + ' end */\n';

                    continue;
                }

                parsed[i] = this.child('link', item.url).process(path);
            }

            return parsed.join('');
        },

        processLink: function(path) {
            return 'url(' + this.pathFrom(path) + ')';
        }

    })
});

function parseUrl(url) {
    if (url.lastIndexOf('url(', 0) === 0) url = url.replace(/^url\(\s*/, '').replace(/\s*\)$/, '');

    if (url.charAt(0) === '\'' || url.charAt(0) === '"') url = url.substr(1, url.length - 2);

    return url;
}

function isRelative(url) {
    return !((url.lastIndexOf('/', 0) === 0) || /^\w+:/.test(url));
}
{'fraction_non_alphanumeric': 0.21872369085704665, 'fraction_numerical': 0.006903519110961442, 'mean_word_length': 1.5548387096774194, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5356701', 'n_tokens_mistral': 1961, 'n_tokens_neox': 1847, 'n_words': 370}