<commit_before>var fs = require('fs');
var path = require('path');
var crypto = require('crypto');
var Buffer = require('buffer').Buffer;
var gutil = require('gulp-util');
var PluginError = gutil.PluginError;
var map = require('event-stream').map;

var revPlugin = function revPlugin(params) {

    var defaultParams = {
        basePath: null,
        regExp: /(?:href=|src=|url\()['|"]([^\s>"']+?)\?rev=(@@@)['|"]/gi
    };
    params = extend({}, defaultParams, params);

    function extend(target) {
        var sources = [].slice.call(arguments, 1);
        sources.forEach(function (source) {
            for (var prop in source) {
                target[prop] = source[prop];
            }
        });
        return target;
    }

    return map(function (file, cb) {
        var contents;
        var lines;
        var i, length;
        var line;
        var groups;
        var dependencyPath;
        var data, hash, hashStr;

        if (!file) {
            throw new PluginError('gulp-rev-append', 'Missing file option for gulp-rev-append.');
        }

        if (!file.contents) {
            throw new PluginError('gulp-rev-append', 'Missing file.contents required for modifying files using gulp-rev-append.');
        }

        contents = file.contents.toString();
        lines = contents.split('\n');
        length = lines.length;

        for (i = 0; i < length; i++) {
            line = lines[i];
            groups = params.regExp.exec(line);

            if (groups && groups.length > 1) {
                // are we an "absoulte path"? (e.g. /js/app.js)
                var normPath = path.normalize(groups[1]);
                if (normPath.indexOf(path.sep) === 0) { //we have absolute path
                    if (params.basePath !== null) {
                        dependencyPath = path.join(params.basePath, normPath);
                    } else {
                        dependencyPath = path.join(file.base, normPath);
                    }
                }
                else {
                    dependencyPath = path.resolve(path.dirname(file.path), normPath);
                }

                try {
                    data = fs.readFileSync(dependencyPath);
                    hash = crypto.createHash('md5');
                    hash.update(data.toString(), 'utf8');
                    hashStr = hash.digest('hex');
                } catch (e) {
                    gutil.log('Can\'t find file (' + dependencyPath + ') from line (' + line.trim() + '). Random string will be used.');
                    hashStr = Math.floor((Math.random() * 10000) + 1);
                }

                line = line.replace(groups[2], hashStr);
            }

            lines[i] = line;
            params.regExp.lastIndex = 0;
        }

        file.contents = new Buffer(lines.join('\n'));
        cb(null, file);

    });

};

module.exports = revPlugin;
<commit_msg>Now plugin can handle 2+ links on 1 line<commit_after>var fs = require('fs');
var path = require('path');
var crypto = require('crypto');
var Buffer = require('buffer').Buffer;
var gutil = require('gulp-util');
var PluginError = gutil.PluginError;
var map = require('event-stream').map;

var revPlugin = function revPlugin(params) {

    var defaultParams = {
        basePath: null,
        regExp: /(?:href=|src=|url\()['|"]([^\s>"']+?)\?rev=(@@@)['|"]/gi
    };
    params = extend({}, defaultParams, params);

    function extend(target) {
        var sources = [].slice.call(arguments, 1);
        sources.forEach(function (source) {
            for (var prop in source) {
                target[prop] = source[prop];
            }
        });
        return target;
    }

    return map(function (file, cb) {
        var contents;
        var lines;
        var i, length;
        var line;
        var groups;
        var dependencyPath;
        var data, hash, hashStr;

        if (!file) {
            throw new PluginError('gulp-rev-append', 'Missing file option for gulp-rev-append.');
        }

        if (!file.contents) {
            throw new PluginError('gulp-rev-append', 'Missing file.contents required for modifying files using gulp-rev-append.');
        }

        contents = file.contents.toString();
        lines = contents.split('\n');
        length = lines.length;

        for (i = 0; i < length; i++) {
            line = lines[i];

            while ((groups = params.regExp.exec(line)) !== null) {
                if (groups.length > 1) {
                    // are we an "absoulte path"? (e.g. /js/app.js)
                    var normPath = path.normalize(groups[1]);
                    if (normPath.indexOf(path.sep) === 0) { //we have absolute path
                        if (params.basePath !== null) {
                            dependencyPath = path.join(params.basePath, normPath);
                        } else {
                            dependencyPath = path.join(file.base, normPath);
                        }
                    }
                    else {
                        dependencyPath = path.resolve(path.dirname(file.path), normPath);
                    }
    
                    try {
                        data = fs.readFileSync(dependencyPath);
                        hash = crypto.createHash('md5');
                        hash.update(data.toString(), 'utf8');
                        hashStr = hash.digest('hex');
                    } catch (e) {
                        gutil.log('Can\'t find file (' + dependencyPath + ') from line (' + line.trim() + '). Random string will be used.');
                        hashStr = Math.floor((Math.random() * 10000) + 1);
                    }
    
                    line = line.replace(groups[2], hashStr);
                }
            }

            lines[i] = line;
            params.regExp.lastIndex = 0;
        }

        file.contents = new Buffer(lines.join('\n'));
        cb(null, file);

    });

};

module.exports = revPlugin;

{'fraction_non_alphanumeric': 0.1401305876443998, 'fraction_numerical': 0.005357441821530219, 'mean_word_length': 1.5249366018596788, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7329337', 'n_tokens_mistral': 1689, 'n_tokens_neox': 1594, 'n_words': 385}