<commit_before>var fs = require('fs');
var path = require('path');
var crypto = require('crypto');
var HTMLProcessor = require('../lib/htmlprocessor');
var lockFile = require('lockfile');
var cleanCSS = require('clean-css');

var proc;
var options;

var hashcatify = function (settings) {

    options = settings;
    options.htmlFile = path.join(process.cwd(),settings.htmlFile);
    options.outputHtmlFile = path.join(process.cwd(),settings.outputHtmlFile);
    console.log(options);
    begin();
};


var begin = function () {

    fs.readFile(options.htmlFile, 'utf8', function (err, data) {

        if(err){
            console.log('Error while reading the file contents: ');
            console.log(err);
            return;
        }

        proc = new HTMLProcessor(path.dirname(options.htmlFile), null, data);
        var sourceHtml = fs.readFileSync(options.htmlFile, {encoding:'utf8'});
        proc.blocks.forEach(function (block) {
            console.log('-----');
            console.log("Processing block for " + block.dest);
            var platform = require('os').platform();
            var prefix = (platform === 'win32') ? 'node ' : '';

            if(block.type=='css')
            {

                try{
                    console.log("Concatenating CSS");
                    var concatenatedCss = '';
                    //Concatenate CSS file contents
                    block.src.forEach(function (css){
                        var fs  = require("fs");
                        concatenatedCss += fs.readFileSync(css).toString();
                    });

                    console.log("Minifying CSS");
                    //Minify using clean-css
                    var minimized = cleanCSS.process(concatenatedCss);
                    fs.writeFileSync(block.dest, minimized);
                }
                catch(e){
                    console.log("ERROR while minifying CSS: ");
                    console.log(e);
                    return;
                }
            }
            else
            {

                try{
                    console.log("Minifying JS");
                    var uglify = require("uglify-js");
                    var uglified = uglify.minify(block.src, {compress:false});
                    fs.writeFileSync(block.dest, uglified.code);
                }
                catch(e){
                    console.log("ERROR while minifying JS: ");
                    console.log(e);
                    return;
                }

            }

            try{
                var fileContents = fs.readFileSync(block.dest, {encoding:'utf8'});

                console.log("Getting hash of file");
                var hash = crypto.createHash('md5').update(fileContents).digest("hex");
                var revPath = (path.join(path.dirname(block.dest), hash.substr(0, 8) + '.' + path.basename(block.dest)));

                console.log("Creating copy with hashed filename");
                fs.createReadStream(block.dest).pipe(fs.createWriteStream(revPath));
            }
            catch(e){
                console.log("ERROR while hashing: ");
                console.log(e);
                return;
            }

            try{
                var blockLine = block.raw.join(proc.linefeed);
                var replacement = blockLine.replace(blockLine, proc.replaceWith(block));
                replacement = replacement.replace(path.basename(block.relativeDest), path.basename(revPath));
                console.log('New reference: ' + replacement);

                lockFile.lockSync('lockfile.lock', {retries:10});

                if(sourceHtml.length <= 0){
                    console.log("ERROR reading " + options.htmlFile + ", it appears to be empty");
                    return;
                }

                sourceHtml = sourceHtml.replace(blockLine, replacement);
            }
            catch(e){
                console.log("ERROR while creating new reference: ");
                console.log(e);
                return;
            }
            finally{
                // Clean up
                lockFile.unlockSync('lockfile.lock');
            }

        });
        console.log("Writing to HTML source file");
        fs.writeFileSync(options.outputHtmlFile, sourceHtml, 'utf8');
        console.log("Hashcat complete")
    });
};


exports.hashcatify = hashcatify;
<commit_msg>Don't write intermediate files; Harmonize quotes<commit_after>var fs = require("fs");
var path = require("path");
var stream = require("stream");
var crypto = require("crypto");
var HTMLProcessor = require("../lib/htmlprocessor");
var lockFile = require("lockfile");
var cleanCSS = require("clean-css");

var proc;
var options;

var hashcatify = function (settings) {

    options = settings;
    options.htmlFile = path.join(process.cwd(),settings.htmlFile);
    options.outputHtmlFile = path.join(process.cwd(),settings.outputHtmlFile);
    console.log(options);
    begin();
};


var begin = function () {

    fs.readFile(options.htmlFile, "utf8", function (err, sourceHtml) {

        if(err){
            console.log("Error while reading the file contents: ");
            console.log(err);
            return;
        }

        proc = new HTMLProcessor(path.dirname(options.htmlFile), null, sourceHtml);
        proc.blocks.forEach(function (block) {
            console.log("-----");
            console.log("Processing block for " + block.dest);
            var platform = require("os").platform();
            var prefix = (platform === "win32") ? "node " : "";
            var fileContents;

            if(block.type=="css")
            {

                try{
                    console.log("Concatenating CSS");
                    var concatenatedCss = "";
                    //Concatenate CSS file contents
                    block.src.forEach(function (css){
                        var fs  = require("fs");
                        concatenatedCss += fs.readFileSync(css).toString();
                    });

                    console.log("Minifying CSS");
                    //Minify using clean-css
                    var minimized = cleanCSS.process(concatenatedCss);
                    fileContents = minimized;
                }
                catch(e){
                    console.log("ERROR while minifying CSS: ");
                    console.log(e);
                    return;
                }
            }
            else
            {

                try{
                    console.log("Minifying JS");
                    var uglify = require("uglify-js");
                    var uglified = uglify.minify(block.src, {compress:false});
                    fileContents = uglified.code;
                }
                catch(e){
                    console.log("ERROR while minifying JS: ");
                    console.log(e);
                    return;
                }

            }

            try{
                console.log("Getting hash of file");
                var hash = crypto.createHash("md5").update(fileContents).digest("hex");
                var revPath = (path.join(path.dirname(block.dest), hash.substr(0, 8) + "." + path.basename(block.dest)));

                console.log("Creating copy with hashed filename");

                var readStream = new stream.Readable();
                readStream.push(fileContents);
                readStream.push(null);
                readStream.pipe(fs.createWriteStream(revPath));
            }
            catch(e){
                console.log("ERROR while hashing: ");
                console.log(e);
                return;
            }

            try{
                var blockLine = block.raw.join(proc.linefeed);
                var replacement = proc.replaceWith(block);
                replacement = replacement.replace(path.basename(block.relativeDest), path.basename(revPath));
                console.log("New reference: " + replacement);

                lockFile.lockSync("lockfile.lock", {retries:10});

                if(sourceHtml.length <= 0){
                    console.log("ERROR reading " + options.htmlFile + ", it appears to be empty");
                    return;
                }

                sourceHtml = sourceHtml.replace(blockLine, replacement);
            }
            catch(e){
                console.log("ERROR while creating new reference: ");
                console.log(e);
                return;
            }
            finally{
                // Clean up
                lockFile.unlockSync("lockfile.lock");
            }

        });
        console.log("Writing to HTML source file");
        fs.writeFileSync(options.outputHtmlFile, sourceHtml, "utf8");
        console.log("Hashcat complete");
    });
};


exports.hashcatify = hashcatify;

{'fraction_non_alphanumeric': 0.12180724259280282, 'fraction_numerical': 0.002497445794074242, 'mean_word_length': 1.4971655328798186, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '870109', 'n_tokens_mistral': 2301, 'n_tokens_neox': 2133, 'n_words': 433}