<commit_before>'use strict';
var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var walkSync = require('walk-sync');
var convert = require('convert-source-map');
var CachingWriter = require('broccoli-caching-writer');

function _genMkdirOrProcess(process) {
	return function(relativePath) {
		if (relativePath.slice(-1) === '/') {
			return mkdirp.sync(path.join(destDir, relativePath));
		} else {
			return process(relativePath);
		};
	}
}

function _isSourceMappableFile(path) {
	return path.slice(-3) === '.js'
		|| path.slice(-4) === '.css';
}

function SourceMapInliner(inputTree) {
	if (!(this instanceof SourceMapInliner)) return new SourceMapInliner(inputTree);
	this.inputTree = inputTree;
}
SourceMapInliner.prototype = Object.create(CachingWriter.prototype);
SourceMapInliner.prototype.constructor = SourceMapInliner;
SourceMapInliner.prototype.updateCache = function(srcDir, destDir) {
	walkSync(srcDir).forEach(_genMkdirOrProcess(function(relativePath) {
		if (_isSourceMappableFile(relativePath)) {
			var srcPath = path.join(srcDir, relativePath);
			var destPath = path.join(destDir, relativePath);
			var srcCode = fs.readFileSync(srcPath, {encoding: 'utf-8'});
			var smap = convert.fromMapFileSource(srcCode, srcDir);
			if (smap !== null) {
				if (typeof smap.getProperty('sourcesContent') === 'undefined' && typeof smap.getProperty('sources') !== 'undefined') {
					var contents = smap.getProperty('sources').map(function(spath) {
						return fs.readFileSync(path.join(srcDir, spath), {encoding: 'utf-8'});
					});
					smap = smap.setProperty('sourcesContent', contents);
				}
				fs.writeFileSync(destPath, convert.removeMapFileComments(srcCode) + '\n' + smap.toComment());
			} else {
				fs.writeFileSync(destPath, srcCode);
			}
		}
	}));
}

function SourceMapExtractor(inputTree) {
	if (!(this instanceof SourceMapExtractor)) return new SourceMapExtractor(inputTree);
	this.inputTree = inputTree;
}
SourceMapExtractor.prototype = Object.create(CachingWriter.prototype);
SourceMapExtractor.prototype.constructor = SourceMapExtractor;
SourceMapExtractor.prototype.updateCache = function(srcDir, destDir) {
	walkSync(srcDir).forEach(_genMkdirOrProcess(function(relativePath) {
		if (_isSourceMappableFile(relativePath)) {
			var srcPath = path.join(srcDir, relativePath);
			var destPath = path.join(destDir, relativePath);
			var srcCode = fs.readFileSync(srcPath, {encoding: 'utf-8'});
			var smap = convert.fromComment(srcCode, srcDir);
			if (smap !== null) {
				fs.writeFileSync(destPath, convert.removeComments(srcCode) + '\n//# sourceMappingURL=' + relativePath + '.map');
				fs.writeFileSync(destPath + '.map', smap.toJSON());
			} else {
				fs.writeFileSync(destPath, srcCode);
			}
		}
	}));
}

module.exports = {
	inline: SourceMapInliner,
	extract: SourceMapExtractor
}<commit_msg>fix css support<commit_after>'use strict';
var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var walkSync = require('walk-sync');
var convert = require('convert-source-map');
var CachingWriter = require('broccoli-caching-writer');

function _genMkdirOrProcess(process) {
	return function(relativePath) {
		if (relativePath.slice(-1) === '/') {
			return mkdirp.sync(path.join(destDir, relativePath));
		} else {
			return process(relativePath);
		}
	}
}

function _isSourceMappableFile(path) {
	return path.slice(-3) === '.js'
		|| path.slice(-4) === '.css';
}

function SourceMapInliner(inputTree) {
	if (!(this instanceof SourceMapInliner)) return new SourceMapInliner(inputTree);
	this.inputTree = inputTree;
}
SourceMapInliner.prototype = Object.create(CachingWriter.prototype);
SourceMapInliner.prototype.constructor = SourceMapInliner;
SourceMapInliner.prototype.updateCache = function(srcDir, destDir) {
	walkSync(srcDir).forEach(_genMkdirOrProcess(function(relativePath) {
		if (_isSourceMappableFile(relativePath)) {
			var srcPath = path.join(srcDir, relativePath);
			var destPath = path.join(destDir, relativePath);
			var srcCode = fs.readFileSync(srcPath, {encoding: 'utf-8'});
			var smap = convert.fromMapFileSource(srcCode, srcDir);
			if (smap !== null && typeof smap['sourcemap'] !== 'undefined') {
				if (typeof smap.getProperty('sourcesContent') === 'undefined' && typeof smap.getProperty('sources') !== 'undefined') {
					var contents = smap.getProperty('sources').map(function(spath) {
						return fs.readFileSync(path.join(srcDir, spath), {encoding: 'utf-8'});
					});
					smap = smap.setProperty('sourcesContent', contents);
				}
				var comment = smap.toComment();
				if (destPath.slice(-4) === '.css') {
					comment = comment.replace(/^\/\//, '/*') + ' */';
				}
				fs.writeFileSync(destPath, convert.removeMapFileComments(srcCode) + '\n' + comment);
			} else {
				fs.writeFileSync(destPath, srcCode);
			}
		}
	}));
}

function SourceMapExtractor(inputTree) {
	if (!(this instanceof SourceMapExtractor)) return new SourceMapExtractor(inputTree);
	this.inputTree = inputTree;
}
SourceMapExtractor.prototype = Object.create(CachingWriter.prototype);
SourceMapExtractor.prototype.constructor = SourceMapExtractor;
SourceMapExtractor.prototype.updateCache = function(srcDir, destDir) {
	walkSync(srcDir).forEach(_genMkdirOrProcess(function(relativePath) {
		if (_isSourceMappableFile(relativePath)) {
			var srcPath = path.join(srcDir, relativePath);
			var destPath = path.join(destDir, relativePath);
			var srcCode = fs.readFileSync(srcPath, {encoding: 'utf-8'});
			var smap = convert.fromComment(srcCode, srcDir);
			if (smap !== null) {
				var comment = '//# sourceMappingURL=' + relativePath + '.map';
				if (destPath.slice(-4) === '.css') {
					comment = '/*# sourceMappingURL=' + relativePath + '.map */';
				}
				fs.writeFileSync(destPath, convert.removeComments(srcCode) + '\n' + comment);
				fs.writeFileSync(destPath + '.map', smap.toJSON());
			} else {
				fs.writeFileSync(destPath, srcCode);
			}
		}
	}));
}

module.exports = {
	inline: SourceMapInliner,
	extract: SourceMapExtractor
}
{'fraction_non_alphanumeric': 0.15303080490228552, 'fraction_numerical': 0.0023186485591255384, 'mean_word_length': 5.14969450101833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5148389', 'n_tokens_mistral': 2139, 'n_tokens_neox': 1937, 'n_words': 349}