<commit_before>/*
 * 
 *
 * Copyright (c) 2013 流火
 * Licensed under the MIT license.
 */

var util = require('util');
var fs = require('fs');
var http = require('http');
var ssi = require('./ssi').ssi,
	ssiChunk = require('./ssi').ssiChunk,
	events = require('events'),
	url  = require('url'),
	path = require('path');

var isUtf8 = require('./is-utf8');
var iconv = require('iconv-lite');
var tidy = require('./tidy');
var extract = require('./extract');
var concat = require('./concat').concat;

module.exports = function(grunt) {

	// Please see the Grunt documentation for more information regarding task
	// creation: http://gruntjs.com/creating-tasks

	grunt.registerMultiTask('relativeurl', 'relativeurl.', function() {
		// Merge task-specific and/or target-specific options with these defaults.
		var options = this.options();

		var that = this;
		var pwd = process.cwd();
		this.files.forEach(function(v,k){
			console.log(v.dest);
			var p = v.src[0];
			var bf = read(p);
			var dirname = path.dirname(v.dest);
			var filep = path.join(dirname,path.basename(v.dest,path.extname(v.dest)));

			// 一定是utf8格式的
			var chunk = ssiChunk(p,bf.toString('utf8'));

            //JS、CSS环境自动切换
            //var relative = '';
            //var base ='http://g.tbcdn.cn';

            //if(options.env === 'daily') base = 'http://g.assets.daily.taobao.net';
            //relative = [base, options.group, options.name, options.version + '/'].join('/');

            var relative = options.relative;

            if(options.combo !== true) {
                chunk = chunk.replace(/(\b(src|href|action))="([^"#:]+)"/gi, '$1="' + relative + '$3"');
                // fix replaced links in the form of "relative/../"
                chunk = chunk.replace(/(\b(src|href|action))="(.+\/)?[^\/]+\/\.\.\//gi, '$1="$3');
            } else {
                var scripts = [];
                var styles = [];

                //chunk = chunk.replace(/<script[\s\S]+?src="([^"]+)"><\/script>/igm, function($, $1) {
                chunk = chunk.replace(/<script[^>]+?src="([^"]+)"><\/script>/igm, function($, $1) {
                    if($.match(/http:\/\//igm)) {
                        return $;
                    }

                    scripts.push($1.replace(/(.+\/)?[^\/]+\/\.\.\//igm, '$1'));

                    if(scripts.length === 1) {
                        return '<%%%SCRIPT_HOLDER%%%>';
                    }

                    return '';
                });

                //chunk = chunk.replace(/<link[\s\S]+?href="([^"]+?)"\s*\/>/igm, function($, $1) {
                chunk = chunk.replace(/<link[^>]+?href="([^"]+?)"[^>]*\/>/igm, function($, $1) {
                    if($.match(/http:\/\//igm)) {
                        return $;
                    }

                    styles.push($1.replace(/(.+\/)?[^\/]+\/\.\.\//igm, '$1'));

                    if(styles.length === 1) {
                        return '<%%%STYLES_HOLDER%%%>';
                    }

                    return '';
                });

                console.log(scripts);
                console.log(styles);

                var combo_script = relative + '??' + scripts.join(',');
                var combo_styles = relative + '??' + styles.join(',');

                chunk = chunk.replace('<%%%SCRIPT_HOLDER%%%>', '<script type="text/javascript" src="' + combo_script + '"></script>');
                chunk = chunk.replace('<%%%STYLES_HOLDER%%%>', '<link rel="stylesheet" href="' + combo_styles + '"/>');
            }

			//Loading "tidy.js" tasks...ERROR
			//>> TypeError: Object #<Object> has no ethod 'charAt'
			//chunk = tidy(chunk);

			if(!(chunk instanceof Buffer)){
				chunk = new Buffer(chunk);
			}
			if(options.encoding == 'gbk'){
				chunk = iconv.encode(iconv.decode(chunk, 'utf8'),'gbk');
			}

			fs.writeFileSync(v.dest,chunk);
			
		});
		return;
	});

};
<commit_msg>removing console.log.<commit_after>/*
 * 
 *
 * Copyright (c) 2013 流火
 * Licensed under the MIT license.
 */

var util = require('util');
var fs = require('fs');
var http = require('http');
var ssi = require('./ssi').ssi,
	ssiChunk = require('./ssi').ssiChunk,
	events = require('events'),
	url  = require('url'),
	path = require('path');

var isUtf8 = require('./is-utf8');
var iconv = require('iconv-lite');
var tidy = require('./tidy');
var extract = require('./extract');
var concat = require('./concat').concat;

module.exports = function(grunt) {

	// Please see the Grunt documentation for more information regarding task
	// creation: http://gruntjs.com/creating-tasks

	grunt.registerMultiTask('relativeurl', 'relativeurl.', function() {
		// Merge task-specific and/or target-specific options with these defaults.
		var options = this.options();

		var that = this;
		var pwd = process.cwd();
		this.files.forEach(function(v,k){
			console.log(v.dest);
			var p = v.src[0];
			var bf = read(p);
			var dirname = path.dirname(v.dest);
			var filep = path.join(dirname,path.basename(v.dest,path.extname(v.dest)));

			// 一定是utf8格式的
			var chunk = ssiChunk(p,bf.toString('utf8'));

            //JS、CSS环境自动切换
            //var relative = '';
            //var base ='http://g.tbcdn.cn';

            //if(options.env === 'daily') base = 'http://g.assets.daily.taobao.net';
            //relative = [base, options.group, options.name, options.version + '/'].join('/');

            var relative = options.relative;

            if(options.combo !== true) {
                chunk = chunk.replace(/(\b(src|href|action))="([^"#:]+)"/gi, '$1="' + relative + '$3"');
                // fix replaced links in the form of "relative/../"
                chunk = chunk.replace(/(\b(src|href|action))="(.+\/)?[^\/]+\/\.\.\//gi, '$1="$3');
            } else {
                var scripts = [];
                var styles = [];

                //chunk = chunk.replace(/<script[\s\S]+?src="([^"]+)"><\/script>/igm, function($, $1) {
                chunk = chunk.replace(/<script[^>]+?src="([^"]+)"><\/script>/igm, function($, $1) {
                    if($.match(/http:\/\//igm)) {
                        return $;
                    }

                    scripts.push($1.replace(/(.+\/)?[^\/]+\/\.\.\//igm, '$1'));

                    if(scripts.length === 1) {
                        return '<%%%SCRIPT_HOLDER%%%>';
                    }

                    return '';
                });

                //chunk = chunk.replace(/<link[\s\S]+?href="([^"]+?)"\s*\/>/igm, function($, $1) {
                chunk = chunk.replace(/<link[^>]+?href="([^"]+?)"[^>]*\/>/igm, function($, $1) {
                    if($.match(/http:\/\//igm)) {
                        return $;
                    }

                    styles.push($1.replace(/(.+\/)?[^\/]+\/\.\.\//igm, '$1'));

                    if(styles.length === 1) {
                        return '<%%%STYLES_HOLDER%%%>';
                    }

                    return '';
                });

                //console.log(scripts);
                //console.log(styles);

                var combo_script = relative + '??' + scripts.join(',');
                var combo_styles = relative + '??' + styles.join(',');

                chunk = chunk.replace('<%%%SCRIPT_HOLDER%%%>', '<script type="text/javascript" src="' + combo_script + '"></script>');
                chunk = chunk.replace('<%%%STYLES_HOLDER%%%>', '<link rel="stylesheet" href="' + combo_styles + '"/>');
            }

			//Loading "tidy.js" tasks...ERROR
			//>> TypeError: Object #<Object> has no ethod 'charAt'
			//chunk = tidy(chunk);

			if(!(chunk instanceof Buffer)){
				chunk = new Buffer(chunk);
			}
			if(options.encoding == 'gbk'){
				chunk = iconv.encode(iconv.decode(chunk, 'utf8'),'gbk');
			}

			fs.writeFileSync(v.dest,chunk);
			
		});
		return;
	});

};

{'fraction_non_alphanumeric': 0.22025641025641027, 'fraction_numerical': 0.006153846153846154, 'mean_word_length': 1.847080291970803, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7351255', 'n_tokens_mistral': 2671, 'n_tokens_neox': 2408, 'n_words': 443}