<commit_before>local ffi = require('ffi')
local utils = require('utils')
local dump = utils.dump
local p = utils.prettyPrint

utils.loadColors(256)
print("256 utils")
p(utils)

utils.loadColors(16)
print("16 utils")
p(utils)

utils.loadColors()
print("no-color")
p(utils)
p(dump(utils))

utils.loadColors(256)

p{"This is \n\r\t cool \0 right?"}
p{1, 2, true, false, nil, "Hello"}

local str = ""
for i = 0, 127 do
  str = str .. string.char(i)
end

p(str)

ffi.cdef[[
typedef struct { uint8_t red, green, blue, alpha; } rgba_pixel;
]]
local img = ffi.new("rgba_pixel[?]", 9)
p{utils.stdout, img, p}

local tim = {name="Tim", age=32}
local jack = {name="Jack", age=8}
p{
  {1,2,3,4},
  {1,2,3,g=5,6,z=2, [10]=10},
}
p{
  [tim] = "programmer",
  [jack] = "player",
}
p{
  programmer = tim,
  player = jack,
}

local cycle = {a="table"}
cycle.self = cycle
p(cycle)
<commit_msg>Make colors test automated<commit_after>local utils = require('utils')
local dump = utils.dump
local strip = utils.strip

require('tap')(function (test)

  test("Recursive values", function ()
    local data = {a="value"}
    data.data = data
    local out = dump(data)
    local stripped = strip(out)
    print("recursive", out, dump(stripped))
    assert(string.match(stripped, "{ a = 'value', data = table: 0x%x+ }"))
  end)

  test("string escapes", function ()
    local tests = {
      '\000\001\002\003\004\005\006\a\b\t\n\v\f\r\014\015',  "'\\000\\001\\002\\003\\004\\005\\006\\a\\b\\t\\n\\v\\f\\r\\014\\015'",
      '\016\017\018\019\020\021\022\023\024\025\026\027\028\029\030\031',  "'\\016\\017\\018\\019\\020\\021\\022\\023\\024\\025\\026\\027\\028\\029\\030\\031'",
      ' !"#$%&\'()*+,-./', '\' !"#$%&\\\'()*+,-./\'',
      '0123456789:;<=>?',  "'0123456789:;<=>?'",
      '@ABCDEFGHIJKLMNO',  "'@ABCDEFGHIJKLMNO'",
      'PQRSTUVWXYZ[\\]^_', "'PQRSTUVWXYZ[\\\\]^_'",
      '`abcdefghijklmno',  "'`abcdefghijklmno'",
      'pqrstuvwxyz{|}',  "'pqrstuvwxyz{|}'",
    }
    for i = 1, 16, 2 do
      local out = dump(tests[i])
      local stripped = strip(out)
      print(out, dump(stripped))
      assert(stripped == tests[i + 1])
    end

  end)

  test("Color mode switching", function ()
    local data = {42,true,"A\nstring"}

    utils.loadColors()
    local plain = dump(data)
    print("plain", plain, dump(plain))
    assert(plain == "{ 42, true, 'A\\nstring' }")

    utils.loadColors(16)
    local colored = dump(data)
    print("colored", colored, dump(colored))
    assert(colored == "\027[1;30m{ \027[0m\027[1;33m42\027[0m\027[1;30m, \027[0m\027[0;33mtrue\027[0m\027[1;30m, \027[0m\027[1;32m'\027[0;32mA\027[1;32m\\n\027[0;32mstring\027[1;32m'\027[0m \027[1;30m}\027[0m")

    utils.loadColors(256)
    local super = dump(data)
    print("super", super, dump(super))
    assert(super == "\027[38;5;247m{ \027[0m\027[38;5;202m42\027[0m\027[38;5;240m, \027[0m\027[38;5;220mtrue\027[0m\027[38;5;240m, \027[0m\027[38;5;40m'\027[38;5;34mA\027[38;5;46m\\n\027[38;5;34mstring\027[38;5;40m'\027[0m \027[38;5;247m}\027[0m")

  end)

end)


{'fraction_non_alphanumeric': 0.23467369808833224, 'fraction_numerical': 0.14634146341463414, 'mean_word_length': 3.5029673590504453, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3180621', 'n_tokens_mistral': 1586, 'n_tokens_neox': 1314, 'n_words': 215}