<commit_before>"use strict";

var async = require('async');
var childProcess = require('child_process');
var storage = require('azure-storage');
var console = require('console');
var fs = require('fs');
var fsextra = require('fs-extra');
var path = require('path');
var process = require('process');
var zlib = require('zlib');

//////////////
//Shared functionality

function loadRemoteAccessInfo() {
    var res = undefined;
    try {
        res = JSON.parse(fs.readFileSync(path.resolve(__dirname, 'azureconfig.json')));
    }
    catch (ex) {
        ;
    }

    return res;
}

function dirLooksLikeTrace(trgtDir) {
    try {
        if (fs.existsSync(trgtDir)) {
            var contents = fs.readdirSync(trgtDir);
            if (contents.length !== 0 && contents.indexOf('ttdlog.log') === -1) {
                //This doesn't look like an old trace directory!
                //We don't want to accidentally blow away user data.
                return false;
            }
        }
    } catch (ex) {
        return false;
    }

    return true;
}

function uploadFileToAzure(localFile, remoteFile, callback) {
    console.log('Uploading data to ' + remoteFile + ' from ' + localFile);

    var accessInfo = loadRemoteAccessInfo();
    if (!accessInfo) {
        callback(new Error('Failed to load Azure config.'));
    }
    else {
        var azureService = storage.createFileService(accessInfo.remoteUser, accessInfo.storageKey);
        azureService.createFileFromLocalFile('cloud-traces', '', remoteFile, localFile, (err, res) => {
            callback(err);
        });
    }
}

function downloadFileFromAzure(remoteFile, localFile, callback) {
    console.log('Downloading data from ' + remoteFile + ' to ' + localFile);

    var accessInfo = loadRemoteAccessInfo();
    if (!accessInfo) {
        callback(new Error('Failed to load Azure config.'));
    }
    else {
        var azureService = storage.createFileService(accessInfo.remoteUser, accessInfo.storageKey);
        azureService.getFileToLocalFile('cloud-traces', '', remoteFile, localFile, (err, res) => {
            callback(err);
        });
    }
}

function removeFileFromAzure(remoteFile, callback) {
    console.log('Removing file from ' + remoteFile);

    var accessInfo = loadRemoteAccessInfo();
    if (!accessInfo) {
        callback(new Error('Failed to load Azure config.'));
    }
    else {
        var azureService = storage.createFileService(accessInfo.remoteUser, accessInfo.storageKey);
        azureService.deleteFileIfExists('cloud-traces', '', remoteFile, (err) => {
            callback(err);
        });
    }
}

//////////////
//Compression functionality

function traceCompressor(traceDir, targetFile, completeCallBack) {
    fs.open(targetFile, "w", (openerr, outFD) => {
        if (openerr) { completeCallBack(openerr); }

        fs.readdir(traceDir, (direrr, compressFiles) => {
            if (direrr) { completeCallBack(direrr); }

            const headerblockLength = compressFiles.reduce((value, file) => {
                return value + (file.length + 16 + 16);
            }, 0);

            fs.write(outFD, headerblockLength.toString() + "%", (herr, initialheaderpos) => {
                if (herr) { completeCallBack(herr); }

                const currentHeaderPos = initialheaderpos;
                const currentDataPos = currentHeaderPos + headerblockLength;

                function writeHeader(file, compressedSize, cb) {
                    fs.stat(file, (serr, stats) => {
                        if (serr) { cb(serr); }

                        const wstr = path.basename(file) + '@' + currentHeaderPos + ':' + compressedSize + '#';
                        currentHeaderPos += wstr.length;
                        fs.write(outFD, wstr, offset, cb);
                    });
                }

                function writeData(file, cb) {
                    const inp = fs.createReadStream(file);
                    const out = fs.createWriteStream(outFile, { flags: "a", start: currentDataPos });

                    out.on('close', () => {
                        currentDataPos += out.bytesWritten;
                        writeHeader(file, out.bytesWritten, cb);
                    });
                    out.on('error', (perr) => {
                        cb(perr);
                    });

                    const defl = zlib.createDeflate();
                    inp.pipe(defl).pipe(out);
                }

                const filecbArray = compressFiles.map((file) => {
                    return function (cb) {
                        writeData(file, cb);
                    }
                });

                async.series(
                    filecbArray,
                    function (err) {
                        fs.close(outFD);
                        completeCallBack(err);
                    }
                );
            });
        });
    });
}

//Run compression of trace dir into temp file
function logCompress(targetFile, traceDirName, callback) {
    console.log('Compressing ' + traceDirName + ' into: ' + targetFile);

    var zipcmd = `node ${dirname}${path.sep}app.js -compress ${traceDirName} -into ${targetFile}`;
    var startTime = new Date();
    childProcess.exec(zipcmd, { env: { DO_TTD_RECORD: 0 } }, (err) => {
        console.log(`Compress complete in ${(new Date() - startTime) / 1000}s.`);

        callback(err);
    });
}
exports.logCompressExecAsync = logCompress;


asdf


function logDecompress(traceFile, traceDirName, callback) {<commit_msg>compress/decompress initial version<commit_after>"use strict";

var async = require('async');
var childProcess = require('child_process');
var storage = require('azure-storage');
var console = require('console');
var fs = require('fs');
var fsextra = require('fs-extra');
var path = require('path');
var process = require('process');
var zlib = require('zlib');

//////////////
//Shared functionality

function loadRemoteAccessInfo() {
    var res = undefined;
    try {
        res = JSON.parse(fs.readFileSync(path.resolve(__dirname, 'azureconfig.json')));
    }
    catch (ex) {
        ;
    }

    return res;
}

function dirLooksLikeTrace(trgtDir) {
    try {
        if (fs.existsSync(trgtDir)) {
            var contents = fs.readdirSync(trgtDir);
            if (contents.length !== 0 && contents.indexOf('ttdlog.log') === -1) {
                //This doesn't look like an old trace directory!
                //We don't want to accidentally blow away user data.
                return false;
            }
        }
    } catch (ex) {
        return false;
    }

    return true;
}

function uploadFileToAzure(localFile, remoteFile, callback) {
    console.log('Uploading data to ' + remoteFile + ' from ' + localFile);

    var accessInfo = loadRemoteAccessInfo();
    if (!accessInfo) {
        callback(new Error('Failed to load Azure config.'));
    }
    else {
        var azureService = storage.createFileService(accessInfo.remoteUser, accessInfo.storageKey);
        azureService.createFileFromLocalFile('cloud-traces', '', remoteFile, localFile, (err, res) => {
            callback(err);
        });
    }
}

function downloadFileFromAzure(remoteFile, localFile, callback) {
    console.log('Downloading data from ' + remoteFile + ' to ' + localFile);

    var accessInfo = loadRemoteAccessInfo();
    if (!accessInfo) {
        callback(new Error('Failed to load Azure config.'));
    }
    else {
        var azureService = storage.createFileService(accessInfo.remoteUser, accessInfo.storageKey);
        azureService.getFileToLocalFile('cloud-traces', '', remoteFile, localFile, (err, res) => {
            callback(err);
        });
    }
}

function removeFileFromAzure(remoteFile, callback) {
    console.log('Removing file from ' + remoteFile);

    var accessInfo = loadRemoteAccessInfo();
    if (!accessInfo) {
        callback(new Error('Failed to load Azure config.'));
    }
    else {
        var azureService = storage.createFileService(accessInfo.remoteUser, accessInfo.storageKey);
        azureService.deleteFileIfExists('cloud-traces', '', remoteFile, (err) => {
            callback(err);
        });
    }
}

//////////////
//Compression functionality

const headerEntrySize = 32 + 32 + 32; //name startpos length\n

function traceCompressor(traceDir, targetFile, completeCallBack) {
    fs.readdir(traceDir, (direrr, compressFiles) => {
        if (direrr) { return completeCallBack(direrr); }

        const headerblockLength = 32 + compressFiles.length * headerEntrySize;
        let headerInfo = compressFiles.length.toString();
        while (headerInfo.length < 32) {
            headerInfo += ' ';
        }
        headerInfo += '\n';

        function extendHeader(file, startPos, length) {
            hval = file + ' ' + startPos + ' ' + length;
            assert(hval < headerEntrySize);

            while (hval.length < headerEntrySize) {
                hval += ' ';
            }
            hval += '\n';

            headerInfo += hval;
        }

        function writeFinalHeaders(file, cb) {
            fs.open(targetFile, 'a', (wfherr, fd) => {
                if (wfherr) { return completeCallBack(wfherr); }

                const headerBuff = new Buffer(headerInfo);
                fs.write(fd, headerBuff, completeCallBack);
            });
        }

        let currentDataPos = headerblockLength;
        fs.writeFile(targetFile, new Buffer(headerblockLength), (ierr) => {
            if (ierr) { return completeCallBack(ierr); }

            function writeData(file, cb) {
                const inp = fs.createReadStream(file);
                const out = fs.createWriteStream(outFile, { flags: "a" });

                out.on('close', () => {
                    extendHeader(file, currentDataPos, out.bytesWritten);
                    currentDataPos += out.bytesWritten;
                    cb(null);
                });
                out.on('error', (perr) => {
                    cb(perr);
                });

                const defl = zlib.createDeflate();
                inp.pipe(defl).pipe(out);
            }

            const filecbArray = compressFiles.map((file) => {
                return function (cb) {
                    writeData(file, cb);
                }
            });

            async.series(
                filecbArray,
                function (err) {
                    if (err) { return completeCallBack(err); }

                    writeFinalHeaders(targetFile, completeCallBack);
                }
            );
        });
    });
}
exports.traceCompressorDirect = traceCompressor;

//Run compression of trace dir into temp file
function logCompress(targetFile, traceDirName, callback) {
    console.log('Compressing ' + traceDirName + ' into: ' + targetFile);

    var zipcmd = `node ${dirname}${path.sep}app.js -compress ${traceDirName} -into ${targetFile}`;
    var startTime = new Date();
    childProcess.exec(zipcmd, { env: { DO_TTD_RECORD: 0 } }, (err) => {
        console.log(`Compress complete in ${(new Date() - startTime) / 1000}s.`);

        callback(err);
    });
}
exports.logCompressExecAsync = logCompress;


asdf

function traceDecompressor(traceFile, targetDir, completeCallBack) {
    function extractHeaderInfo(cb) {
        fs.open(traceFile, 'r', (oerr, fd) => {
            if (oerr) { return completeCallBack(oerr); }

            const psizeBuff = new Buffer(32);
            fs.read(fd, psizeBuff, 0, psizeBuff.length, 0, (sizeerr, sizebytes, sizebuff) => {
                if (sizeerr) { return cb(sizeerr); }

                const headerblockLength = Number.parseInt(sizebuff.toString());
                if (headerblockLength === NaN) { return cb(new Error('Failed to parse header info')); }

                const pheadersBuff = new Buffer(headerblockLength);
                fs.read(fd, pheadersBuff, 0, pheadersBuff.length, 0, (herr, headersBytes, headersBuff) => {
                    if (herr) { return cb(herr); }

                    const headers = headersBuff.toString().split('\n').shift().map((headerStr) => {
                        const components = headerStr.split(/\s+/);
                        return { file: components[0], startOffset: components[1], length: components[2] };
                    });

                    fs.close(fs, (cerr) => {
                        if (sizeerr) { return cb(cerr); }
                        cb(null, headers);
                    });
                });
            });
        });
    }

    function extractFile(headerInfo, cb) {
        const inp = fs.createReadStream(traceFile, { start: headerInfo.startOffset, end: headerInfo.startOffset + headerInfo.length - 1 });
        const out = fs.createWriteStream(path.join(targetDir, headerInfo.file));

        out.on('close', () => {
            cb(null);
        });
        out.on('error', (perr) => {
            cb(perr);
        });

        const defl = zlib.createDeflate();
        inp.pipe(defl).pipe(out);
    }

    extractHeaderInfo((err, headers) => {
        const filecbArray = headers.map((header) => {
            return function (cb) {
                extractFile(header, cb);
            }
        });

        async.series(
            filecbArray,
            function (err) {
                return completeCallBack(err);
            }
        );
    });
}
exports.traceDecompressorDirect = traceDecompressor;

function logDecompress(traceFile, traceDirName, callback) {
{'fraction_non_alphanumeric': 0.13028977988586674, 'fraction_numerical': 0.0028903876083895353, 'mean_word_length': 2.0053452115812918, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7256567', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3635, 'n_words': 798}