<commit_before>import { ipcRenderer, remote } from 'electron';
import fh from './helpers/fileHelpers';
import jetpack from 'fs-jetpack';

window.xelib = remote.getGlobal('xelib');

let alphabet = 'abcdefghijklmnopqrstuvwxyz';

let deserialize = function(str) {
    return JSON.parse(str, function(key, value) {
        if (key === '' || (typeof value !== 'string')) return value;

        let expr = /function[^\(]*\(([^\)]*)\)[^\{]*{([^\}]*)\}/,
            match = value.match(expr);
        if (!match) return value;

        let args = match[1].split(',').map(function(arg) {
            return arg.replace(/\s+/, '');
        });
        return new Function(args, match[2]);
    });
};

let buildCallbacksCode = function(callbacks) {
    return callbacks.map(function(key) {
        return `let ${key} = (...args) => invokeCallback('${key}', args);`;
    }).join('\r\n');
};

let codeSection = function(label, code) {
    return [`// ${label.toUpperCase()}`, code, ''].join('\r\n');
};

let buildWorkerCode = function(options) {
    return [
        codeSection('callbacks', buildCallbacksCode(options.callbacks)),
        codeSection('worker', jetpack.read(options.filename))
    ].join('\r\n');
};

let invokeCallback = function(callbackName, args) {
    ipcRenderer.send('worker-callback', {
        callbackName: callbackName,
        args: args
    });
};

let log = function(message) {
    ipcRenderer.send('worker-message', message);
};

let runWorker = function(payload) {
    let options = deserialize(payload);
    if (!jetpack.exists(options.filename))
        throw new Error(`File ${options.filename} not found in ${jetpack.cwd()}`);

    log(`Starting worker: ${options.filename}`);
    let workerCode = buildWorkerCode(options),
        workerFn = new Function('data', 'fh', 'invokeCallback', 'log', workerCode);
    log(workerCode);
    workerFn(options.data, fh, invokeCallback, log);
    log('Worker done!');
};

window.onload = function () {
    ipcRenderer.on('run-worker', (e, payload) => {
        try {
            runWorker(payload);
            ipcRenderer.send('worker-done');
        } catch(e) {
            log(e.stack);
            ipcRenderer.send('worker-error', e.stack);
        }
    });
};<commit_msg>Now I know my ABCs, next time won't you sing with me!  :abc:<commit_after>import { ipcRenderer, remote } from 'electron';
import fh from './helpers/fileHelpers';
import jetpack from 'fs-jetpack';

window.xelib = remote.getGlobal('xelib');

let deserialize = function(str) {
    return JSON.parse(str, function(key, value) {
        if (key === '' || (typeof value !== 'string')) return value;

        let expr = /function[^\(]*\(([^\)]*)\)[^\{]*{([^\}]*)\}/,
            match = value.match(expr);
        if (!match) return value;

        let args = match[1].split(',').map(function(arg) {
            return arg.replace(/\s+/, '');
        });
        return new Function(args, match[2]);
    });
};

let buildCallbacksCode = function(callbacks) {
    return callbacks.map(function(key) {
        return `let ${key} = (...args) => invokeCallback('${key}', args);`;
    }).join('\r\n');
};

let codeSection = function(label, code) {
    return [`// ${label.toUpperCase()}`, code, ''].join('\r\n');
};

let buildWorkerCode = function(options) {
    return [
        codeSection('callbacks', buildCallbacksCode(options.callbacks)),
        codeSection('worker', jetpack.read(options.filename))
    ].join('\r\n');
};

let invokeCallback = function(callbackName, args) {
    ipcRenderer.send('worker-callback', {
        callbackName: callbackName,
        args: args
    });
};

let log = function(message) {
    ipcRenderer.send('worker-message', message);
};

let runWorker = function(payload) {
    let options = deserialize(payload);
    if (!jetpack.exists(options.filename))
        throw new Error(`File ${options.filename} not found in ${jetpack.cwd()}`);

    log(`Starting worker: ${options.filename}`);
    let workerCode = buildWorkerCode(options),
        workerFn = new Function('data', 'fh', 'invokeCallback', 'log', workerCode);
    log(workerCode);
    workerFn(options.data, fh, invokeCallback, log);
    log('Worker done!');
};

window.onload = function () {
    ipcRenderer.on('run-worker', (e, payload) => {
        try {
            runWorker(payload);
            ipcRenderer.send('worker-done');
        } catch(e) {
            log(e.stack);
            ipcRenderer.send('worker-error', e.stack);
        }
    });
};
{'fraction_non_alphanumeric': 0.1866220735785953, 'fraction_numerical': 0.0008918617614269788, 'mean_word_length': 3.05972850678733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3077111', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1396, 'n_words': 291}