<commit_before>// converts a character to a Baudot sequence
// "shift" is "l" or "f" (letter or figure)
var charToBaud = function  ( aChar ) {
    var aTable = {};
    aTable["a"]  = [0,0,0,1,1];
    aTable["b"]  = [1,1,0,0,1];
    aTable["c"]  = [0,1,1,1,0];
    aTable["d"]  = [0,1,0,0,1];
    aTable["e"]  = [0,0,0,0,1];
    aTable["f"]  = [0,1,1,0,1];
    aTable["g"]  = [1,1,0,1,0];
    aTable["h"]  = [1,0,1,0,0];
    aTable["i"]  = [0,0,1,1,0];
    aTable["j"]  = [0,1,0,1,1];
    aTable["k"]  = [0,1,1,1,1];
    aTable["l"]  = [1,0,0,1,0];
    aTable["m"]  = [1,1,1,0,0];
    aTable["n"]  = [0,1,1,0,0];
    aTable["o"]  = [1,1,0,0,0];
    aTable["p"]  = [1,0,1,1,0];
    aTable["q"]  = [1,0,1,1,1];
    aTable["r"]  = [0,1,0,1,0];
    aTable["s"]  = [0,0,1,0,1];
    aTable["t"]  = [1,0,0,0,0];
    aTable["u"]  = [0,0,1,1,1];
    aTable["v"]  = [1,1,1,1,0];
    aTable["w"]  = [1,0,0,1,1];
    aTable["x"]  = [1,1,1,0,1];
    aTable["y"]  = [1,0,1,0,1];
    aTable["z"]  = [1,0,0,0,1];
    aTable["CR"] = [0,1,0,0,0];
    aTable["LF"] = [0,0,0,1,0];
    aTable[" "]  = [0,0,1,0,0];
    aTable["SF"] = [1,1,0,1,1];
    aTable["NL"] = [0,0,0,0,0];
    aTable["ZZ"] = [1,1,1,1,1];
    aTable["-"]  = [0,0,0,1,1];
    aTable["?"]  = [1,1,0,0,1];
    aTable[":"]  = [0,1,1,1,0];
    aTable["$"]  = [0,1,0,0,1];
    aTable["3"]  = [0,0,0,0,1];
    aTable["!"]  = [0,1,1,0,1];
    aTable["&"]  = [1,1,0,1,0];
    aTable["#"]  = [1,0,1,0,0];
    aTable["8"]  = [0,0,1,1,0];
    aTable["'"]  = [0,1,0,1,1];
    aTable["("]  = [0,1,1,1,1];
    aTable[")"]  = [1,0,0,1,0];
    aTable["."]  = [1,1,1,0,0];
    aTable[","]  = [0,1,1,0,0];
    aTable["9"]  = [1,1,0,0,0];
    aTable["0"]  = [1,0,1,1,0];
    aTable["1"]  = [1,0,1,1,1];
    aTable["4"]  = [0,1,0,1,0];
    aTable["BL"] = [0,0,1,0,1];
    aTable["5"]  = [1,0,0,0,0];
    aTable["7"]  = [0,0,1,1,1];
    aTable[";"]  = [1,1,1,1,0];
    aTable["2"]  = [1,0,0,1,1];
    aTable["/"]  = [1,1,1,0,1];
    aTable["6"]  = [1,0,1,0,1];
    aTable["\""] = [1,0,0,0,1];
    aTable["CR"] = [0,1,0,0,0];
    aTable["LF"] = [0,0,0,1,0];
    aTable[" "]  = [0,0,1,0,0];
    aTable["ZZ"] = [1,1,0,1,1];
    aTable["NL"] = [0,0,0,0,0];
    aTable["SL"] = [1,1,1,1,1];

    return aTable[aChar];

};

// converts Baudot code to character
var baudToChar = function( baudArray, shift ) {
    
    var lTable = {};
    lTable[[0,0,0,1,1]] = "a";
    lTable[[1,1,0,0,1]] = "b";
    lTable[[0,1,1,1,0]] = "c";
    lTable[[0,1,0,0,1]] = "d";
    lTable[[0,0,0,0,1]] = "e";
    lTable[[0,1,1,0,1]] = "f";
    lTable[[1,1,0,1,0]] = "g";
    lTable[[1,0,1,0,0]] = "h";
    lTable[[0,0,1,1,0]] = "i";
    lTable[[0,1,0,1,1]] = "j";
    lTable[[0,1,1,1,1]] = "k";
    lTable[[1,0,0,1,0]] = "l";
    lTable[[1,1,1,0,0]] = "m";
    lTable[[0,1,1,0,0]] = "n";
    lTable[[1,1,0,0,0]] = "o";
    lTable[[1,0,1,1,0]] = "p";
    lTable[[1,0,1,1,1]] = "q";
    lTable[[0,1,0,1,0]] = "r";
    lTable[[0,0,1,0,1]] = "s";
    lTable[[1,0,0,0,0]] = "t";
    lTable[[0,0,1,1,1]] = "u";
    lTable[[1,1,1,1,0]] = "v";
    lTable[[1,0,0,1,1]] = "w";
    lTable[[1,1,1,0,1]] = "x";
    lTable[[1,0,1,0,1]] = "y";
    lTable[[1,0,0,0,1]] = "z";
    lTable[[0,1,0,0,0]] = "CR";
    lTable[[0,0,0,1,0]] = "LF";
    lTable[[0,0,1,0,0]] = " ";
    lTable[[1,1,0,1,1]] = "SF";
    lTable[[0,0,0,0,0]] = "NL";
    lTable[[1,1,1,1,1]] = "ZZ";

    var fTable = {};
    fTable[[0,0,0,1,1]] = "-";
    fTable[[1,1,0,0,1]] = "?";
    fTable[[0,1,1,1,0]] = ":";
    fTable[[0,1,0,0,1]] = "$";
    fTable[[0,0,0,0,1]] = "3";
    fTable[[0,1,1,0,1]] = "!";
    fTable[[1,1,0,1,0]] = "&";
    fTable[[1,0,1,0,0]] = "#";
    fTable[[0,0,1,1,0]] = "8";
    fTable[[0,1,0,1,1]] = "'";
    fTable[[0,1,1,1,1]] = "(";
    fTable[[1,0,0,1,0]] = ")";
    fTable[[1,1,1,0,0]] = ".";
    fTable[[0,1,1,0,0]] = ",";
    fTable[[1,1,0,0,0]] = "9";
    fTable[[1,0,1,1,0]] = "0";
    fTable[[1,0,1,1,1]] = "1";
    fTable[[0,1,0,1,0]] = "4";
    fTable[[0,0,1,0,1]] = "BL";
    fTable[[1,0,0,0,0]] = "5";
    fTable[[0,0,1,1,1]] = "7";
    fTable[[1,1,1,1,0]] = ";";
    fTable[[1,0,0,1,1]] = "2";
    fTable[[1,1,1,0,1]] = "/";
    fTable[[1,0,1,0,1]] = "6";
    fTable[[1,0,0,0,1]] = "\"";
    fTable[[0,1,0,0,0]] = "CR";
    fTable[[0,0,0,1,0]] = "LF";
    fTable[[0,0,1,0,0]] = " ";
    fTable[[1,1,0,1,1]] = "ZZ";
    fTable[[0,0,0,0,0]] = "NL";
    fTable[[1,1,1,1,1]] = "SL";

    if(shift === "l") {
        return lTable[baudArray]; }
    else if(shift === "f") {
        return fTable[baudArray]; }
    else { throw "incorrect shift"; }
};

var stringToBaudArray = function( inString ) {
    var letters = "abcdefghijklmnopqrstuvwxyz";
    var figures = "-?:$3!&#8'().,901457;2/6\" ";
    var baudArray = [];
    var shift = "l"; // starts in letter position
    for (var i = 0; i < inString.length; i++) {
        var l = inString[i];

        // if letter, but shift is in figure position
        if (letters.indexOf(l) != -1 && shift != "l") {
            shift = "l";
            baudArray.push( [1,1,1,1,1] );
        }<commit_msg>Reversed Baudot code bit order.<commit_after>// converts a character to a Baudot sequence
// "shift" is "l" or "f" (letter or figure)
var charToBaud = function  ( aChar ) {
    var aTable = {};
    aTable["a"]  = [1,1,0,0,0];
    aTable["b"]  = [1,0,0,1,1];
    aTable["c"]  = [0,1,1,1,0];
    aTable["d"]  = [1,0,0,1,0];
    aTable["e"]  = [1,0,0,0,0];
    aTable["f"]  = [1,0,1,1,0];
    aTable["g"]  = [0,1,0,1,1];
    aTable["h"]  = [0,0,1,0,1];
    aTable["i"]  = [0,1,1,0,0];
    aTable["j"]  = [1,1,0,1,0];
    aTable["k"]  = [1,1,1,1,0];
    aTable["l"]  = [0,1,0,0,1];
    aTable["m"]  = [0,0,1,1,1];
    aTable["n"]  = [0,0,1,1,0];
    aTable["o"]  = [0,0,0,1,1];
    aTable["p"]  = [0,1,1,0,1];
    aTable["q"]  = [1,1,1,0,1];
    aTable["r"]  = [0,1,0,1,0];
    aTable["s"]  = [1,0,1,0,0];
    aTable["t"]  = [0,0,0,0,1];
    aTable["u"]  = [1,1,1,0,0];
    aTable["v"]  = [0,1,1,1,1];
    aTable["w"]  = [1,1,0,0,1];
    aTable["x"]  = [1,0,1,1,1];
    aTable["y"]  = [1,0,1,0,1];
    aTable["z"]  = [1,0,0,0,1];
    aTable["CR"] = [0,0,0,1,0];
    aTable["LF"] = [0,1,0,0,0];
    aTable[" "]  = [0,0,1,0,0];
    aTable["SF"] = [1,1,0,1,1];
    aTable["NL"] = [0,0,0,0,0];
    aTable["ZZ"] = [1,1,1,1,1];
    aTable["-"]  = [1,1,0,0,0];
    aTable["?"]  = [1,0,0,1,1];
    aTable[":"]  = [0,1,1,1,0];
    aTable["$"]  = [1,0,0,1,0];
    aTable["3"]  = [1,0,0,0,0];
    aTable["!"]  = [1,0,1,1,0];
    aTable["&"]  = [0,1,0,1,1];
    aTable["#"]  = [0,0,1,0,1];
    aTable["8"]  = [0,1,1,0,0];
    aTable["'"]  = [1,1,0,1,0];
    aTable["("]  = [1,1,1,1,0];
    aTable[")"]  = [0,1,0,0,1];
    aTable["."]  = [0,0,1,1,1];
    aTable[","]  = [0,0,1,1,0];
    aTable["9"]  = [0,0,0,1,1];
    aTable["0"]  = [0,1,1,0,1];
    aTable["1"]  = [1,1,1,0,1];
    aTable["4"]  = [0,1,0,1,0];
    aTable["BL"] = [1,0,1,0,0];
    aTable["5"]  = [0,0,0,0,1];
    aTable["7"]  = [1,1,1,0,0];
    aTable[";"]  = [0,1,1,1,1];
    aTable["2"]  = [1,1,0,0,1];
    aTable["/"]  = [1,0,1,1,1];
    aTable["6"]  = [1,0,1,0,1];
    aTable["\""] = [1,0,0,0,1];
    aTable["CR"] = [0,0,0,1,0];
    aTable["LF"] = [0,1,0,0,0];
    aTable[" "]  = [0,0,1,0,0];
    aTable["ZZ"] = [1,1,0,1,1];
    aTable["NL"] = [0,0,0,0,0];
    aTable["SL"] = [1,1,1,1,1];

    return aTable[aChar];

};

// converts Baudot code to character
var baudToChar = function( baudArray, shift ) {
    
    var lTable = {};
    lTable[[1,1,0,0,0]] = "a";
    lTable[[1,0,0,1,1]] = "b";
    lTable[[0,1,1,1,0]] = "c";
    lTable[[1,0,0,1,0]] = "d";
    lTable[[1,0,0,0,0]] = "e";
    lTable[[1,0,1,1,0]] = "f";
    lTable[[0,1,0,1,1]] = "g";
    lTable[[0,0,1,0,1]] = "h";
    lTable[[0,1,1,0,0]] = "i";
    lTable[[1,1,0,1,0]] = "j";
    lTable[[1,1,1,1,0]] = "k";
    lTable[[0,1,0,0,1]] = "l";
    lTable[[0,0,1,1,1]] = "m";
    lTable[[0,0,1,1,0]] = "n";
    lTable[[0,0,0,1,1]] = "o";
    lTable[[0,1,1,0,1]] = "p";
    lTable[[1,1,1,0,1]] = "q";
    lTable[[0,1,0,1,0]] = "r";
    lTable[[1,0,1,0,0]] = "s";
    lTable[[0,0,0,0,1]] = "t";
    lTable[[1,1,1,0,0]] = "u";
    lTable[[0,1,1,1,1]] = "v";
    lTable[[1,1,0,0,1]] = "w";
    lTable[[1,0,1,1,1]] = "x";
    lTable[[1,0,1,0,1]] = "y";
    lTable[[1,0,0,0,1]] = "z";
    lTable[[0,0,0,1,0]] = "CR";
    lTable[[0,1,0,0,0]] = "LF";
    lTable[[0,0,1,0,0]] = " ";
    lTable[[1,1,0,1,1]] = "SF";
    lTable[[0,0,0,0,0]] = "NL";
    lTable[[1,1,1,1,1]] = "ZZ";

    var fTable = {};
    fTable[[1,1,0,0,0]] = "-";
    fTable[[1,0,0,1,1]] = "?";
    fTable[[0,1,1,1,0]] = ":";
    fTable[[1,0,0,1,0]] = "$";
    fTable[[1,0,0,0,0]] = "3";
    fTable[[1,0,1,1,0]] = "!";
    fTable[[0,1,0,1,1]] = "&";
    fTable[[0,0,1,0,1]] = "#";
    fTable[[0,1,1,0,0]] = "8";
    fTable[[1,1,0,1,0]] = "'";
    fTable[[1,1,1,1,0]] = "(";
    fTable[[0,1,0,0,1]] = ")";
    fTable[[0,0,1,1,1]] = ".";
    fTable[[0,0,1,1,0]] = ",";
    fTable[[0,0,0,1,1]] = "9";
    fTable[[0,1,1,0,1]] = "0";
    fTable[[1,1,1,0,1]] = "1";
    fTable[[0,1,0,1,0]] = "4";
    fTable[[1,0,1,0,0]] = "BL";
    fTable[[0,0,0,0,1]] = "5";
    fTable[[1,1,1,0,0]] = "7";
    fTable[[0,1,1,1,1]] = ";";
    fTable[[1,1,0,0,1]] = "2";
    fTable[[1,0,1,1,1]] = "/";
    fTable[[1,0,1,0,1]] = "6";
    fTable[[1,0,0,0,1]] = "\"";
    fTable[[0,0,0,1,0]] = "CR";
    fTable[[0,1,0,0,0]] = "LF";
    fTable[[0,0,1,0,0]] = " ";
    fTable[[1,1,0,1,1]] = "ZZ";
    fTable[[0,0,0,0,0]] = "NL";
    fTable[[1,1,1,1,1]] = "SL";

    if(shift === "l") {
        return lTable[baudArray]; }
    else if(shift === "f") {
        return fTable[baudArray]; }
    else { throw "incorrect shift"; }
};

var stringToBaudArray = function( inString ) {
    var letters = "abcdefghijklmnopqrstuvwxyz";
    var figures = "-?:$3!&#8'().,901457;2/6\" ";
    var baudArray = [];
    var shift = "l"; // starts in letter position
    for (var i = 0; i < inString.length; i++) {
        var l = inString[i];

        // if letter, but shift is in figure position
        if (letters.indexOf(l) != -1 && shift != "l") {
            shift = "l";
            baudArray.push( [1,1,1,1,1] );
        }
{'fraction_non_alphanumeric': 0.34241823587710607, 'fraction_numerical': 0.13419226957383548, 'mean_word_length': 3.0139220365950674, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3537628', 'n_tokens_mistral': 5661, 'n_tokens_neox': 5622, 'n_words': 676}