<commit_before>local ffi = require "ffi"
local lunit = require "lunitx"
local capnp = require "capnp"
local util = require "util"
local hw_capnp = require "handwritten_capnp"
--local format = string.format

local tdiff = util.table_diff


if _VERSION >= 'Lua 5.2' then
    _ENV = lunit.module('simple','seeall')
else
    module( "simple", package.seeall, lunit.testcase )
end

local copy = {}

function test_basic_value()
    local data = {
        i0 = 32,
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(data.i0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value1()
    local data = {
        b0 = true,
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(true, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value2()
    local data = {
        i2 = -8,
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(-8, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value3()
    local data = {
        s0 = {},
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_not_nil(copy.s0)
    assert_equal(0, copy.s0.f0)
    assert_equal(0, copy.s0.f1)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value4()
    local data = {
        s0 = {
            f0 = 3.14,
            f1 = 3.1415926535
        },
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_not_nil(copy.s0)
    -- assert_equal(3.1400001049042, copy.s0.f0)
    assert_equal(3.1415926535, copy.s0.f1)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value4()
    local data = {
        l0 = { 1, -1, 127 }
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_equal(3, #copy.l0)
    assert_equal(1, copy.l0[1])
    assert_equal(-1, copy.l0[2])
    assert_equal(127, copy.l0[3])
    assert_nil(copy.t0)
end

function test_basic_value5()
    local data = {
        t0 = "1234567890~!#$%^&*()-=_+[]{};':|,.<>/?"
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_equal(38, #copy.t0)
    assert_equal("1234567890~!#$%^&*()-=_+[]{};':|,.<>/?", copy.t0)
end<commit_msg>Add more test cases for handwritten encoding and decoding<commit_after>local ffi = require "ffi"
local lunit = require "lunitx"
local capnp = require "capnp"
local util = require "util"
local hw_capnp = require "handwritten_capnp"
--local format = string.format

local tdiff = util.table_diff


if _VERSION >= 'Lua 5.2' then
    _ENV = lunit.module('simple','seeall')
else
    module( "simple", package.seeall, lunit.testcase )
end

local copy = {}

function test_basic_value()
    local data = {
        i0 = 32,
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(data.i0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value1()
    local data = {
        b0 = true,
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(true, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value2()
    local data = {
        i2 = -8,
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(-8, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value3()
    local data = {
        s0 = {},
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_not_nil(copy.s0)
    assert_equal(0, copy.s0.f0)
    assert_equal(0, copy.s0.f1)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value4()
    local data = {
        s0 = {
            f0 = 3.14,
            f1 = 3.1415926535
        },
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_not_nil(copy.s0)
    -- assert_equal(3.1400001049042, copy.s0.f0)
    assert_equal(3.1415926535, copy.s0.f1)
    assert_nil(copy.l0)
    assert_nil(copy.t0)
end

function test_basic_value4()
    local data = {
        l0 = { 1, -1, 127 }
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_equal(3, #copy.l0)
    assert_equal(1, copy.l0[1])
    assert_equal(-1, copy.l0[2])
    assert_equal(127, copy.l0[3])
    assert_nil(copy.t0)
end

function test_basic_value5()
    local data = {
        t0 = "1234567890~!#$%^&*()-=_+[]{};':|,.<>/?"
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(0, copy.i0)
    assert_equal(0, copy.i1)
    assert_equal(0, copy.i2)
    assert_equal(false, copy.b0)
    assert_equal(false, copy.b1)
    assert_equal(0, copy.i3)
    assert_equal("enum1", copy.e0)
    assert_equal("enum5", copy.e1)
    assert_nil(copy.s0)
    assert_nil(copy.l0)
    assert_equal(38, #copy.t0)
    assert_equal("1234567890~!#$%^&*()-=_+[]{};':|,.<>/?", copy.t0)
end

function test_basic_value6()
    local data = {
        i0 = 32,
        i1 = 16,
        i2 = 127,
        b0 = true,
        b1 = true,
        i3 = 65536,
        e0 = "enum3",
        s0 = {
            f0 = 3.14,
            f1 = 3.14159265358979,
        },
        l0 = { 28, 29 },
        t0 = "hello",
        e1 = "enum7",
    }

    local bin   = hw_capnp.T1.serialize(data)
    copy  = hw_capnp.T1.parse(bin, copy)
    assert_equal(32, copy.i0)
    assert_equal(16, copy.i1)
    assert_equal(127, copy.i2)
    assert_equal(true, copy.b0)
    assert_equal(true, copy.b1)
    assert_equal(65536, copy.i3)
    assert_equal("enum3", copy.e0)
    assert_equal("enum7", copy.e1)
    assert_not_nil(copy.s0)
    -- assert_equal(3.14, copy.s0.f0)
    assert_equal(3.14159265358979, copy.s0.f1)
    assert_not_nil(copy.l0)
    assert_equal(2, #copy.l0)
    assert_equal(28, copy.l0[1])
    assert_equal(29, copy.l0[2])
    assert_equal(5, #copy.t0)
    assert_equal("hello", copy.t0)
end
{'fraction_non_alphanumeric': 0.142425505313607, 'fraction_numerical': 0.0619920816836841, 'mean_word_length': 2.8503810669875653, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6112962', 'n_tokens_mistral': 4147, 'n_tokens_neox': 3988, 'n_words': 650}