<commit_before>local String = require("string");
local Array = require("lockbox.util.array");
local Stream = require("lockbox.util.stream");
local Base64 = require("lockbox.util.base64");

local test_vectors = {
    {
        plain = "foobar",
        enc = "Zm9vYmFy"
    },
    {
        plain = "fooba",
        enc = "Zm9vYmE="
    },
    {
        plain = "foob",
        enc = "Zm9vYg=="
    },
    {
        plain = "foo",
        enc = "Zm9v"
    },
    {
        plain = "fo",
        enc = "Zm8="
    },
    {
        plain = "f",
        enc = "Zg=="
    },
    {
        plain = "",
        enc = ""
    },


};

for k, v in pairs(test_vectors) do
    local out = Base64.fromStream(Stream.fromString(v.plain));
    assert(out == v.enc, String.format("STREAM ENCODING FAILED expected(%s) actual(%s)", v.enc, out));

    local out = Base64.fromString(v.plain);
    assert(out == v.enc, String.format("ARRAY ENECODING FAILED expected(%s) actual(%s)", v.enc, out));

    local out = Base64.toArray(v.enc);
    out = Array.toString(out);

    assert(out == v.plain, String.format("ARRAY DECODING FAILED expected(%s) actual(%s)", v.plain, out));

    local out = Base64.toString(v.enc);
    assert(out == v.plain, String.format("ARRAY DECODING FAILED expected(%s) actual(%s)", v.plain, out));

end<commit_msg>Added missing 'fromArray' and 'toStream' tests for Base64.<commit_after>local String = require("string");
local Array = require("lockbox.util.array");
local Stream = require("lockbox.util.stream");
local Base64 = require("lockbox.util.base64");

local test_vectors = {
    {
        plain = "foobar",
        enc = "Zm9vYmFy"
    },
    {
        plain = "fooba",
        enc = "Zm9vYmE="
    },
    {
        plain = "foob",
        enc = "Zm9vYg=="
    },
    {
        plain = "foo",
        enc = "Zm9v"
    },
    {
        plain = "fo",
        enc = "Zm8="
    },
    {
        plain = "f",
        enc = "Zg=="
    },
    {
        plain = "",
        enc = ""
    },


};

for k, v in pairs(test_vectors) do
    local out = Base64.fromStream(Stream.fromString(v.plain));
    assert(out == v.enc, String.format("STREAM ENCODING FAILED expected(%s) actual(%s)", v.enc, out));

    local out = Base64.fromArray(Array.fromString(v.plain));
    assert(out == v.enc, String.format("STREAM ENCODING FAILED expected(%s) actual(%s)", v.enc, out));

    local out = Base64.fromString(v.plain);
    assert(out == v.enc, String.format("ARRAY ENECODING FAILED expected(%s) actual(%s)", v.enc, out));

    local out = Base64.toStream(v.enc);
    out = Stream.toString(out);
    assert(out == v.plain, String.format("ARRAY DECODING FAILED expected(%s) actual(%s)", v.plain, out));

    local out = Base64.toArray(v.enc);
    out = Array.toString(out);
    assert(out == v.plain, String.format("ARRAY DECODING FAILED expected(%s) actual(%s)", v.plain, out));

    local out = Base64.toString(v.enc);
    assert(out == v.plain, String.format("ARRAY DECODING FAILED expected(%s) actual(%s)", v.plain, out));

end
{'fraction_non_alphanumeric': 0.17594433399602386, 'fraction_numerical': 0.013253810470510271, 'mean_word_length': 2.598331346841478, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2445384', 'n_tokens_mistral': 1090, 'n_tokens_neox': 1040, 'n_words': 226}