<commit_before>var INHERIT = require('inherit'),
    base = require('../tech'),
    cssbase = require('./css-base'),

    stringRe = "(?:(?:'[^'\\r\\n]*')|(?:\"[^\"\\r\\n]*\"))",
    urlRe = "(?:(?:url\\(\\s*" + stringRe + "\\s*\\))|(?:url\\(\\s*[^\\s\\r\\n'\"]*\\s*\\)))",
    commentRe = '(?:/\\*[^*]*\\*+(?:[^/][^*]*\\*+)*/)',
    importRe = '(?:\\@import\\s+(' + urlRe + '|' + stringRe + '))',
    allRe = new RegExp(commentRe + '|' + importRe + '|' + urlRe, 'g'),
    urlStringRx = new RegExp('^' + urlRe + '$');

exports.Tech = INHERIT(cssbase.Tech, {
    File: INHERIT(cssbase.File, {

        parseInclude: function(content) {
            var m, found = [];

            if (Buffer.isBuffer(content)) content = content.toString('utf8');

            while (m = allRe.exec(content)) {
                if (m[0].lastIndexOf('/*', 0) === 0) {
                    // skip comment
                } else if (m[0].charAt(0) === '@') {
                    // @import
                    var url = parseUrl(m[1]);
                    if (isProcessable(url)) found.push({ type: 'include', url: url, range: [ m.index, allRe.lastIndex ] });
                } else if (urlStringRx.test(m[0])) {
                    // url(...)
                    var url = parseUrl(m[0]);
                    if (isProcessable(url)) found.push({ type: 'link', url: url, range: [ m.index, allRe.lastIndex-1 ] });
                } else {
                    throw new Error('Failed to match: ' + m[0]);
                }
            }

            return makeParsed(found, content);
        },

        processInclude: function(path, content) {
            var parsed = content || this.content;

            for(var i = 0; i < parsed.length; i++) {
                var item = parsed[i];

                if (typeof item === 'string') continue;

                if (item.type === 'include') {
                    parsed[i] = '/* ' + item.url + ' begin */\n' +
                        this.child('include', item.url).process(path) +
                        '\n/* ' + item.url + ' end */\n';

                    continue;
                }

                parsed[i] = this.child('link', item.url).process(path);
            }

            return parsed.join('');
        },

        processLink: function(path) {
            return 'url(' + this.__base(path) + ')';
        },

        processPath: function(path) {
            return path.replace(/^(.*?)(\?|$)/, '$1');
        }

    })
});

function parseUrl(url) {
    if (url.lastIndexOf('url(', 0) === 0) url = url.replace(/^url\(\s*/, '').replace(/\s*\)$/, '');

    if (url.charAt(0) === '\'' || url.charAt(0) === '"') url = url.substr(1, url.length - 2);

    return url;
}

function isProcessable(url) {
    if (url.indexOf('#') === 0 || url.indexOf('?') === 0) return false;
    return !((url.lastIndexOf('/', 0) === 0) || /^\w+:/.test(url));
}

function makeParsed(items, content) {
    var result = [],
        lastInd = 0;

    items.forEach(function(item) {
        if (lastInd > item.range[0]) throw 'index out of range';

        if (lastInd < item.range[0]) result.push(content.substring(lastInd, item.range[0]));

        result.push(item);
        lastInd = item.range[1] + 1;
    });

    if (lastInd < content.length) result.push(content.substring(lastInd));

    return result;
}
<commit_msg>css-fast tech skips includes with absolute paths (Close #47)<commit_after>var INHERIT = require('inherit'),
    base = require('../tech'),
    cssbase = require('./css-base'),

    stringRe = "(?:(?:'[^'\\r\\n]*')|(?:\"[^\"\\r\\n]*\"))",
    urlRe = "(?:(?:url\\(\\s*" + stringRe + "\\s*\\))|(?:url\\(\\s*[^\\s\\r\\n'\"]*\\s*\\)))",
    commentRe = '(?:/\\*[^*]*\\*+(?:[^/][^*]*\\*+)*/)',
    importRe = '(?:\\@import\\s+(' + urlRe + '|' + stringRe + '))',
    allRe = new RegExp(commentRe + '|' + importRe + '|' + urlRe, 'g'),
    urlStringRx = new RegExp('^' + urlRe + '$');

exports.Tech = INHERIT(cssbase.Tech, {
    File: INHERIT(cssbase.File, {

        parseInclude: function(content) {
            var m, found = [];

            if (Buffer.isBuffer(content)) content = content.toString('utf8');

            while (m = allRe.exec(content)) {
                if (m[0].lastIndexOf('/*', 0) === 0) {
                    // skip comment
                } else if (m[0].charAt(0) === '@') {
                    // @import
                    var url = parseUrl(m[1]);
                    if (isIncludeProcessable(url)) found.push({
                        type: 'include',
                        url: url,
                        range: [m.index, allRe.lastIndex]
                    });
                } else if (urlStringRx.test(m[0])) {
                    // url(...)
                    var url = parseUrl(m[0]);
                    if (isLinkProcessable(url)) found.push({
                        type: 'link',
                        url: url,
                        range: [m.index, allRe.lastIndex - 1]
                    });
                } else {
                    throw new Error('Failed to match: ' + m[0]);
                }
            }

            return makeParsed(found, content);
        },

        processInclude: function(path, content) {
            var parsed = content || this.content;

            for(var i = 0; i < parsed.length; i++) {
                var item = parsed[i];

                if (typeof item === 'string') continue;

                if (item.type === 'include') {
                    parsed[i] = '/* ' + item.url + ' begin */\n' +
                        this.child('include', item.url).process(path) +
                        '\n/* ' + item.url + ' end */\n';

                    continue;
                }

                parsed[i] = this.child('link', item.url).process(path);
            }

            return parsed.join('');
        },

        processLink: function(path) {
            return 'url(' + this.__base(path) + ')';
        },

        processPath: function(path) {
            return path.replace(/^(.*?)(\?|$)/, '$1');
        }

    })
});

function parseUrl(url) {
    if (url.lastIndexOf('url(', 0) === 0) url = url.replace(/^url\(\s*/, '').replace(/\s*\)$/, '');

    if (url.charAt(0) === '\'' || url.charAt(0) === '"') url = url.substr(1, url.length - 2);

    return url;
}

function isLinkProcessable(url) {
    return !(~['#', '?', '/'].indexOf(url.charAt(0)) || isAbsoluteUrl(url));
}

function isIncludeProcessable(url) {
    return !isAbsoluteUrl(url);
}

function isAbsoluteUrl(url) {
    return /^\w+:/.test(url);
}

function makeParsed(items, content) {
    var result = [],
        lastInd = 0;

    items.forEach(function(item) {
        if (lastInd > item.range[0]) throw 'index out of range';

        if (lastInd < item.range[0]) result.push(content.substring(lastInd, item.range[0]));

        result.push(item);
        lastInd = item.range[1] + 1;
    });

    if (lastInd < content.length) result.push(content.substring(lastInd));

    return result;
}

{'fraction_non_alphanumeric': 0.21257142857142858, 'fraction_numerical': 0.008142857142857143, 'mean_word_length': 1.6438821752265862, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6573541', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2172, 'n_words': 422}