<commit_before>-- Lua script used to clean up tabs and spaces in C, CPP and H files.
-- Copyright (c) 2014, bel
-- MIT License (http://opensource.org/licenses/mit-license.php)
--
-- It can be used from the command line:
-- Call Lua5.1 or Lua5.2 + this script file + the C/CPP/H file to clean
--
-- It can be used in Visual Studio as an external tool:
-- command: Lua5.1.exe or Lua5.2.exe
-- argument: "X:\civetweb\resources\cleanup.lua" $(ItemPath)
--

clean = arg[1]
print("Cleaning " .. clean)

lines = io.lines(clean)
if not lines then
    print("Can not open file " .. clean)
    return
end

function trimright(s)
  return s:match "^(.-)%s*$"
end

local lineend = false
local tabspace = false
local changed = false
local invalid = false
local newfile = {}

for l in lines do
    local lt = trimright(l)
    if (lt ~= l) then
        lineend = true
        changed = true
    end
    local lts = lt:gsub('\t', '    ')
    if (lts ~= lt) then
        tabspace = true
        changed = true
    end
    for i=1,#lts do
        local b = string.byte(lts,i)
        if b<32 or b>=127 then
            print("Letter " .. string.byte(l,i) .. " (" .. b .. ") found in line " .. lts)
            invalid = true
        end
    end

    newfile[#newfile + 1] = lts
end

print("Line endings trimmed:     " .. tostring(lineend))
print("Tabs converted to spaces: " .. tostring(tabspace))
print("Invalid characters:       " .. tostring(invalid))

if changed then
    local f = io.open(clean, "w")
    for i=1,#newfile do
        f:write(newfile[i])
        f:write("\n")
    end
    f:close()
    print("File cleaned")
end<commit_msg>Do not change line endings to \r\n on Windows<commit_after>-- Lua script used to clean up tabs and spaces in C, CPP and H files.
-- Copyright (c) 2014, bel
-- MIT License (http://opensource.org/licenses/mit-license.php)
--
-- It can be used from the command line:
-- Call Lua5.1 or Lua5.2 + this script file + the C/CPP/H file to clean
--
-- It can be used in Visual Studio as an external tool:
-- command: Lua5.1.exe or Lua5.2.exe
-- argument: "X:\civetweb\resources\cleanup.lua" $(ItemPath)
--

clean = arg[1]
print("Cleaning " .. clean)

lines = io.lines(clean)
if not lines then
    print("Can not open file " .. clean)
    return
end

function trimright(s)
  return s:match "^(.-)%s*$"
end

local lineend = false
local tabspace = false
local changed = false
local invalid = false
local newfile = {}

for l in lines do
    local lt = trimright(l)
    if (lt ~= l) then
        lineend = true
        changed = true
    end
    local lts = lt:gsub('\t', '    ')
    if (lts ~= lt) then
        tabspace = true
        changed = true
    end
    for i=1,#lts do
        local b = string.byte(lts,i)
        if b<32 or b>=127 then
            print("Letter " .. string.byte(l,i) .. " (" .. b .. ") found in line " .. lts)
            invalid = true
        end
    end

    newfile[#newfile + 1] = lts
end

print("Line endings trimmed:     " .. tostring(lineend))
print("Tabs converted to spaces: " .. tostring(tabspace))
print("Invalid characters:       " .. tostring(invalid))

if changed then
    local f = io.open(clean, "wb")
    for i=1,#newfile do
        f:write(newfile[i])
        f:write("\n")
    end
    f:close()
    print("File cleaned")
end
{'fraction_non_alphanumeric': 0.13715330691862238, 'fraction_numerical': 0.012800975312404754, 'mean_word_length': 2.3185035389282103, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2843487', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1109, 'n_words': 371}