<commit_before><!DOCTYPE html>
<head>
  <meta charset="utf-8">
  <title>Lexeduct</title>
</head>
<body>

<header>
  <h1>Lexeduct</h1>
</header>

<div id="container"></div>

</body>
<script src="lexeduct-browser.js"></script>
<script src="yoob/element-factory.js"></script>
<script>
"use strict";

var compose = function(g, f) {
    return function(str, data) {
        str = f(str, data);
        return g(str, data);
    };
};

var container, input, output, processButton, tranformersPanel;

container = document.getElementById('container');
input = yoob.makeTextArea(container, 40, 20, "Hello, there!\nWhat do you think?");

var transformersPanel = yoob.makeDiv(container);
transformersPanel.style.border = "2px solid black";
transformersPanel.style.display = "inline-block";
transformersPanel.style.verticalAlign = "top";

var MAX_TRANSFORMER_SLOTS = 8; // TODO dynamic
var transformerSlots = [];

var transformerNames = [["identity", "---"]];
for (var key in transformer) {
    if (key !== 'identity' && transformer.hasOwnProperty(key)) {
        transformerNames.push([key, key]);
    }
}

var parseOptions = function(text) {
    var args = text.split(' ');
    var cfg = {};
    for (var i = 0; i < args.length; i++) {
        var paramPair = args[i].split('=');
        if (paramPair.length == 2) {
            cfg[paramPair[0]] = paramPair[1];
        } else {
            // register an error
        }
    }
    return cfg;
};

var process = function() {
    var t = transformer['identity'].makeTransformer({});
    for (var i = 0; i < transformerSlots.length; i++) {
        var transformerName = transformerSlots[i].name;
        var selectedParams = transformerSlots[i].selectedParams;
        var t2 = transformer[transformerName].makeTransformer(selectedParams);
        t = compose(t2, t);
    }
    var inLines = input.value.split('\n');
    var outLines = [];
    for (var i = 0; i < inLines.length; i++) {
        outLines.push(t(inLines[i]));
    }
    output.value = outLines.join('\n');
};

processButton = yoob.makeButton(transformersPanel, "Process", process);

var liveMode;
yoob.makeCheckbox(transformersPanel, false, "Live mode", function(b) {
    liveMode = b;
});
yoob.makeLineBreak(transformersPanel);

var updateParametersPanel = function(panel, parameters) {
    panel.innerHTML = "";  // delete any previous controls
    for (var key in parameters) {
        if (parameters.hasOwnProperty(key)) {
            var label = yoob.makeSpan(panel, key);
            label.style.width = "50%";
            var input = yoob.makeTextInput(panel, 24);
            input.style.width = "50%";
            // TODO: populate it with the default
            input.onchange = function() {
                alert('TODO: update transformerSlots.selectedParameters or smth here');
                if (liveMode) {
                    process();
                }
            }
            yoob.makeLineBreak(panel);
        }
    }
};

var makeTransformerSlot = function(container, index) {
    var select = yoob.makeSelect(
        transformersPanel, "Transformer " + (index+1), transformerNames
    );
    yoob.makeLineBreak(transformersPanel);
    var parametersPanel = yoob.makeDiv(transformersPanel);
    
    var getSelectedParams = function(panel) {
        // TODO FIXME
        return parseOptions(transformerSlots[i].optionsInput.value);
    };

    select.onchange = function(e) {
        updateParametersPanel(parametersPanel);
        transformerSlots[index].name = select.options[select.selectedIndex].value;
        transformerSlots[index].selectedParams = getSelectedParams(parametersPanel);
        if (liveMode) {
            process();
        }
    };

    //------------ deprecated...
    var optionsInput = yoob.makeTextInput(transformersPanel, 24);
    optionsInput.onchange = function(e) {
        if (liveMode) {
            process();
        }
    };
    optionsInput.title = "Options";
    yoob.makeLineBreak(transformersPanel);
    //------------

    return {
        select: select,
        optionsInput: optionsInput,
        parametersPanel: parametersPanel
    };
};

for (var i = 0; i < MAX_TRANSFORMER_SLOTS; i++) {
    var slot = makeTransformerSlot(transformersPanel, index);
    transformerSlots.push(slot);
}

output = yoob.makeTextArea(container, 40, 20);
</script><commit_msg>optionsInput is gone now and it sort of actually works!<commit_after><!DOCTYPE html>
<head>
  <meta charset="utf-8">
  <title>Lexeduct</title>
</head>
<body>

<header>
  <h1>Lexeduct</h1>
</header>

<div id="container"></div>

</body>
<script src="lexeduct-browser.js"></script>
<script src="yoob/element-factory.js"></script>
<script>
"use strict";

var compose = function(g, f) {
    return function(str, data) {
        str = f(str, data);
        return g(str, data);
    };
};

var container, input, output, processButton, tranformersPanel;

container = document.getElementById('container');
input = yoob.makeTextArea(container, 40, 20, "Hello, there!\nWhat do you think?");

var transformersPanel = yoob.makeDiv(container);
transformersPanel.style.border = "2px solid black";
transformersPanel.style.display = "inline-block";
transformersPanel.style.verticalAlign = "top";

var MAX_TRANSFORMER_SLOTS = 8; // TODO dynamic
var transformerSlots = [];

var transformerNames = [["identity", "---"]];
for (var key in transformer) {
    if (key !== 'identity' && transformer.hasOwnProperty(key)) {
        transformerNames.push([key, key]);
    }
}

var parseOptions = function(text) {
    var args = text.split(' ');
    var cfg = {};
    for (var i = 0; i < args.length; i++) {
        var paramPair = args[i].split('=');
        if (paramPair.length == 2) {
            cfg[paramPair[0]] = paramPair[1];
        } else {
            // register an error
        }
    }
    return cfg;
};

var process = function() {
    var t = transformer['identity'].makeTransformer({});
    for (var i = 0; i < transformerSlots.length; i++) {
        console.log(uneval(transformerSlots[i]));
        var transformerName = transformerSlots[i].name;
        var selectedParams = transformerSlots[i].selectedParams;
        var t2 = transformer[transformerName].makeTransformer(selectedParams);
        t = compose(t2, t);
    }
    var inLines = input.value.split('\n');
    var outLines = [];
    for (var i = 0; i < inLines.length; i++) {
        outLines.push(t(inLines[i]));
    }
    output.value = outLines.join('\n');
};

processButton = yoob.makeButton(transformersPanel, "Process", process);

var liveMode;
yoob.makeCheckbox(transformersPanel, false, "Live mode", function(b) {
    liveMode = b;
});
yoob.makeLineBreak(transformersPanel);

var updateParametersPanel = function(slot, panel) {
    var parameters = transformer[slot.name].parameters;
    panel.innerHTML = "";  // delete any previous controls
    for (var key in parameters) {
        if (parameters.hasOwnProperty(key)) {
            var label = yoob.makeSpan(panel, key);
            label.style.width = "50%";
            var input = yoob.makeTextInput(panel, 24);
            input.style.width = "50%";
            // TODO: populate it with the default
            // TODO: and assign it to slot.selectedParams[key]
            input.onchange = function() {
                slot.selectedParams[key] = input.value;
                if (liveMode) {
                    process();
                }
            }
            yoob.makeLineBreak(panel);
        }
    }
};

var makeTransformerSlot = function(container, index) {
    var select = yoob.makeSelect(
        transformersPanel, "Transformer " + (index+1), transformerNames
    );
    yoob.makeLineBreak(transformersPanel);
    var parametersPanel = yoob.makeDiv(transformersPanel);
    
    select.onchange = function(e) {
        transformerSlots[index].name = select.options[select.selectedIndex].value;
        updateParametersPanel(transformerSlots[index], parametersPanel);
        if (liveMode) {
            process();
        }
    };

    return {
        name: 'identity',
        selectedParams: {},
        select: select,
        parametersPanel: parametersPanel
    };
};

for (var i = 0; i < MAX_TRANSFORMER_SLOTS; i++) {
    var slot = makeTransformerSlot(transformersPanel, i);
    transformerSlots.push(slot);
}

output = yoob.makeTextArea(container, 40, 20);
</script>
{'fraction_non_alphanumeric': 0.14260014391940512, 'fraction_numerical': 0.007195970256656272, 'mean_word_length': 2.8234754699679048, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467496', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2501, 'n_words': 556}