<commit_before>require("io")
require("string")
socket = require("socket")
-- open the CLI interface.
local commands = {}
commands.auto = nil


local client = nil -- client object for connection
local running = true


-- Connect to remote server. 2 arguments: address, port
function commands.connect(cmd, addr, port)
	assert(addr)
	assert(port)
	print("Connecting to "..addr..":"..port.."...")
	
	client,err = socket.connect(addr, port)
	if not client then
		print("TCP object creation failed:"..err)
		print("Connect failed")
		return nil
	end
	print("Connected.")
end

-- Help command
function commands.help()
	print("Available commands:")
	for k,v in pairs(commands) do
		print(k,v)
	end
end

-- breaks string str into an array of words
local function words(rest)
	local words = {}
	local i = 1 -- lua uses 1-indexed arrays
	local idx=0

	while idx do
		idx = string.find(rest, " ")
		if idx then
			local word = string.sub(rest, 1, idx-1)
			rest = string.sub(rest, idx+1)
			words[i] = word
			i=i+1
		else
			if string.len(rest)>0 then
				words[i] = rest
			end
		end
	end
	return words
end


local function testwords()
	retv = words("word1 word2 word3 test6")
	for k,v in ipairs(retv) do
		print(k,v)
		print(string.len(v))
	end
	print(tostring(retv))
	assert(words("word1 word2 word3 test6")=={"word1","word2","word3","test6"},"Failed 'words' test case 1")
	assert(words("")=={}, "Failed 'words' test case 2")
	assert(words("word1")=={"word1"}, "Failed 'words' test case 3")
	assert(words("word1 ")=={"word1"}, "Failed 'words' test case 4")
end
-- run some tests
--testwords()

-- main loop. read commands and lookup into function table.
while running do
	-- read one line from stdio
	local cmdline = io.read()
	local cmdwords = words(cmdline)
	local cmd = cmdwords[1]
	commands[cmd](unpack(cmdwords))
	
end
	<commit_msg>moved 'words' function to util<commit_after>require("io")
require("string")
util = require("util.lua")
socket = require("socket")
-- open the CLI interface.
local commands = {}
commands.auto = nil


local client = nil -- client object for connection
local running = true


-- Connect to remote server. 2 arguments: address, port
function commands.connect(cmd, addr, port)
	assert(addr)
	assert(port)
	print("Connecting to "..addr..":"..port.."...")
	
	client,err = socket.connect(addr, port)
	if not client then
		print("TCP object creation failed:"..err)
		print("Connect failed")
		return nil
	end
	print("Connected.")
end

-- Help command
function commands.help()
	print("Available commands:")
	for k,v in pairs(commands) do
		print(k,v)
	end
end

-- main loop. read commands and lookup into function table.
while running do
	-- read one line from stdio
	local cmdline = io.read()
	local cmdwords = util.words(cmdline)
	local cmd = cmdwords[1]
	commands[cmd](unpack(cmdwords))
	
end
	
{'fraction_non_alphanumeric': 0.14204545454545456, 'fraction_numerical': 0.011363636363636364, 'mean_word_length': 4.021390374331551, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4676582', 'n_tokens_mistral': 980, 'n_tokens_neox': 921, 'n_words': 306}