<commit_before>local lunit = require "lunit"
local tutil = require "utils"
local TEST_CASE, skip = tutil.TEST_CASE, tutil.skip

local IS_WINDOWS = package.config:sub(1,1) == '\\'
local fs, wcs, _T, _t

local function B(str)
  return (str:gsub(".", function(ch)
    return string.format("\\%.03d", ch:byte())
  end))
end

local function Q(str)
  return string.format("%q",str)
end

local function pass_thrue(str) return str end

local function prequire(...)
  local ok, mod = pcall(require, ...)
  if not ok then return nil, mod end
  return mod
end

local function clone(t, o)
  o = o or {}
  for k,v in pairs(t) do
    o[ k ] = v
  end
  return o
end

local function up(str)
  return IS_WINDOWS and str:upper() or str
end

local function ifind(t, f)
  for k, v in ipairs(t) do
    if up(v) == up(f) then
      return k
    end
  end
end

local function CREATE_TEST(name)

local fs, _T, _t = fs, _T, _t
local DIR_SEP = fs.DIR_SEP

local function mkfile(P, data)
  local f, e = io.open(_t(P), "w+b")
  if not f then return nil, e end
  if data then assert(f:write(data)) end
  f:close()
  return P
end

local function read_file(P)
  local f, err = io.open(_t(P), "rb")
  if not f then return nil, err end
  local data, err = f:read("*all")
  f:close()
  if data then return data end
  return nil, err
end

local function J(...)
  return (table.concat({...}, DIR_SEP))
end

local _ENV = TEST_CASE(name .. ": basic") do

local cwd

function setup()
  cwd = fs.currentdir()
end

function teardown()
  fs.chdir(cwd)
  fs.rmdir(J(cwd, _T"1"))
end

function test_cwd()
  assert_string(cwd)
  assert_equal(cwd, fs.exists(cwd))
  assert_equal(cwd, fs.isdir(cwd) )
  assert_false(fs.isfile(cwd))
end

function test_md()
  local P = J(cwd, _T"1")
  assert_false(fs.exists(P))
  assert_true (fs.mkdir(P))
  assert_nil  (fs.mkdir(P))
  assert_equal(P, fs.exists(P))
  assert_true (fs.chdir(P))
  assert_equal(P, fs.currentdir())
  assert_true (fs.chdir(cwd))
  assert_equal(cwd, fs.currentdir())
  assert_true (fs.rmdir(P))
  assert_false(fs.exists(P))
end

end

local _ENV = TEST_CASE(name .. ": file manipulation") do

local cwd, base
local data ="123\r\n456"

function teardown()
  fs.chdir(cwd)
  fs.remove(J(base, _T"test.txt"))
  fs.remove(J(base, _T"test2.txt"))
  fs.rmdir(base)
end

function setup()
  cwd = fs.currentdir()
  base = J(cwd, _T"tmp")

  teardown()
  assert_true(fs.mkdir(base))
  assert(mkfile(J(base, _T"test.txt"), data))
end

function test_remove()
  assert_true(fs.remove(J(base, _T"test.txt")))
  assert_nil(fs.remove(J(base, _T"test.txt")))
end

function test_size()
  assert_equal(#data, fs.size(J(base, _T"test.txt")))
end

function test_touch()
  local SRC = J(base, _T"test.txt")
  local t1 = assert_number(fs.mtime(SRC)) + 100<commit_msg>Test detect read/write text mode in copy function.<commit_after>local lunit = require "lunit"
local tutil = require "utils"
local TEST_CASE, skip = tutil.TEST_CASE, tutil.skip

local IS_WINDOWS = package.config:sub(1,1) == '\\'
local fs, wcs, _T, _t

local function B(str)
  return (str:gsub(".", function(ch)
    return string.format("\\%.03d", ch:byte())
  end))
end

local function Q(str)
  return string.format("%q",str)
end

local function pass_thrue(str) return str end

local function prequire(...)
  local ok, mod = pcall(require, ...)
  if not ok then return nil, mod end
  return mod
end

local function clone(t, o)
  o = o or {}
  for k,v in pairs(t) do
    o[ k ] = v
  end
  return o
end

local function up(str)
  return IS_WINDOWS and str:upper() or str
end

local function ifind(t, f)
  for k, v in ipairs(t) do
    if up(v) == up(f) then
      return k
    end
  end
end

local function CREATE_TEST(name)

local fs, _T, _t = fs, _T, _t
local DIR_SEP = fs.DIR_SEP

local function mkfile(P, data)
  local f, e = io.open(_t(P), "w+b")
  if not f then return nil, e end
  if data then assert(f:write(data)) end
  f:close()
  return P
end

local function read_file(P)
  local f, err = io.open(_t(P), "rb")
  if not f then return nil, err end
  local data, err = f:read("*all")
  f:close()
  if data then return data end
  return nil, err
end

local function J(...)
  return (table.concat({...}, DIR_SEP))
end

local _ENV = TEST_CASE(name .. ": basic") do

local cwd

function setup()
  cwd = fs.currentdir()
end

function teardown()
  fs.chdir(cwd)
  fs.rmdir(J(cwd, _T"1"))
end

function test_cwd()
  assert_string(cwd)
  assert_equal(cwd, fs.exists(cwd))
  assert_equal(cwd, fs.isdir(cwd) )
  assert_false(fs.isfile(cwd))
end

function test_md()
  local P = J(cwd, _T"1")
  assert_false(fs.exists(P))
  assert_true (fs.mkdir(P))
  assert_nil  (fs.mkdir(P))
  assert_equal(P, fs.exists(P))
  assert_true (fs.chdir(P))
  assert_equal(P, fs.currentdir())
  assert_true (fs.chdir(cwd))
  assert_equal(cwd, fs.currentdir())
  assert_true (fs.rmdir(P))
  assert_false(fs.exists(P))
end

end

local _ENV = TEST_CASE(name .. ": file manipulation") do

local cwd, base
local data ="123\r\n456\n789"

function teardown()
  fs.chdir(cwd)
  fs.remove(J(base, _T"test.txt"))
  fs.remove(J(base, _T"test2.txt"))
  fs.rmdir(base)
end

function setup()
  cwd = fs.currentdir()
  base = J(cwd, _T"tmp")

  teardown()
  assert_true(fs.mkdir(base))
  assert(mkfile(J(base, _T"test.txt"), data))
end

function test_remove()
  assert_true(fs.remove(J(base, _T"test.txt")))
  assert_nil(fs.remove(J(base, _T"test.txt")))
end

function test_size()
  assert_equal(#data, fs.size(J(base, _T"test.txt")))
end

function test_touch()
  local SRC = J(base, _T"test.txt")
  local t1 = assert_number(fs.mtime(SRC)) + 100
{'fraction_non_alphanumeric': 0.15012585400934916, 'fraction_numerical': 0.0066522833513124775, 'mean_word_length': 3.9273693534100973, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1951797', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2255, 'n_words': 600}