<commit_before>strung = require"strung"

print('testing pattern matching')

function f(s, p)
  local i,e = strung.find(s, p)
  if i then return string.sub(s, i, e) end
end

function f1(s, p)
  p = strung.gsub(p, "%%([0-9])", function (s) return "%" .. (s+1) end)
  p = strung.gsub(p, "^(^?)", "%1()", 1)
  p = strung.gsub(p, "($?)$", "()%1", 1)
  local t = {strung.match(s, p)}
  return strung.sub(s, t[1], t[#t] - 1)
end

a,b = strung.find('', '')    -- empty patterns are tricky
assert(a == 1 and b == 0);
a,b = strung.find('alo', '')
assert(a == 1 and b == 0)
a,b = strung.find('a\0o a\0o a\0o', 'a', 1)   -- first position
assert(a == 1 and b == 1)
a,b = strung.find('a\0o a\0o a\0o', 'a\0o', 2)   -- starts in the midle
assert(a == 5 and b == 7)
a,b = strung.find('a\0o a\0o a\0o', 'a\0o', 9)   -- starts in the midle
assert(a == 9 and b == 11)
a,b = strung.find('a\0a\0a\0a\0\0ab', '\0ab', 2);  -- finds at the end
assert(a == 9 and b == 11);
a,b = strung.find('a\0a\0a\0a\0\0ab', 'b')    -- last position
assert(a == 11 and b == 11)
assert(strung.find('a\0a\0a\0a\0\0ab', 'b\0') == nil)   -- check ending
assert(strung.find('', '\0') == nil)
assert(strung.find('alo123alo', '12') == 4)
assert(strung.find('alo123alo', '^12') == nil)

assert(f('aloALO', '%l*') == 'alo')
assert(f('aLo_ALO', '%a*') == 'aLo')

assert(f('aaab', 'a*') == 'aaa');
assert(f('aaa', '^.*$') == 'aaa');
assert(f('aaa', 'b*') == '');
assert(f('aaa', 'ab*a') == 'aa')
assert(f('aba', 'ab*a') == 'aba')
assert(f('aaab', 'a+') == 'aaa')
assert(f('aaa', '^.+$') == 'aaa')
assert(f('aaa', 'b+') == nil)
assert(f('aaa', 'ab+a') == nil)
assert(f('aba', 'ab+a') == 'aba')
assert(f('a$a', '.$') == 'a')
assert(f('a$a', '.%$') == 'a$')
assert(f('a$a', '.$.') == 'a$a')
assert(f('a$a', '$$') == nil)
assert(f('a$b', 'a$') == nil)
assert(f('a$a', '$') == '')
assert(f('', 'b*') == '')
assert(f('aaa', 'bb*') == nil)
assert(f('aaab', 'a-') == '')
assert(f('aaa', '^.-$') == 'aaa')
assert(f('aabaaabaaabaaaba', 'b.*b') == 'baaabaaabaaab')
assert(f('aabaaabaaabaaaba', 'b.-b') == 'baaab')
assert(f('alo xo', '.o$') == 'xo')
assert(f(' \n isto ? assim', '%S%S*') == 'isto')
assert(f(' \n isto ? assim', '%S*$') == 'assim')
assert(f(' \n isto ? assim', '[a-z]*$') == 'assim')
assert(f('um caracter ? extra', '[^%sa-z]') == '?')
assert(f('', 'a?') == '')
assert(f('?', '??') == '?')
assert(f('?bl', '??b?l?') == '?bl')
assert(f('  ?bl', '??b?l?') == '')
assert(f('aa', '^aa?a?a') == 'aa')
assert(f(']]]?b', '[^]]') == '?')
assert(f("0alo alo", "%x*") == "0a")
assert(f("alo alo", "%C+") == "alo alo")
print('+')

-- assert(f1('alo alx 123 b\0o b\0o', '(..*) %1') == "b\0o b\0o")
-- assert(f1('axz123= 4= 4 34', '(.+)=(.*)=%2 %1') == '3= 4= 4 3')
-- assert(f1('=======', '^(=*)=%1$') == '=======')
assert(strung.match('==========', '^([=]*)=%1$') == nil)

local function range (i, j)
  if i <= j then
    return i, range(i+1, j)
  end
end

local abc = string.char(range(0, 255));

assert(string.len(abc) == 256)

local function strset (p)
  local res = {s=''}
  strung.gsub(abc, p, function (c)
    -- print"===="
    -- print("c:", c)
    -- print("type(c):", type(c))
    -- print("c:byte()",c:byte(1))
    -- print("#c:", #c)
    -- print("bef res.s:", res.s)
    -- print("bef #res.s:", #res.s)
    -- print("bef type(res.s):", type(res.s))
    res.s = res.s .. c
    -- print("aft res:",res.s)
    -- print(type(res.s))
    -- print(#res.s)
  end)
  -- print(type(res.s))
  -- print(#res.s)
  return res.s
end;

assert(string.len(strset('[\110-\120]')) == 11)
assert(string.len(strset('[\200-\210]')) == 11)

assert(strset('[a-z]') == "abcdefghijklmnopqrstuvwxyz")
assert(strset('[a-z%d]') == strset('[%da-uu-z]'))
assert(strset('[a-]') == "-a")
assert(strset('[^%W]') == strset('[%w]'))
assert(strset('[]%%]') == '%]')
assert(strset('[a%-z]') == '-az')
assert(strset('[%^%[%-a%]%-b]') == '-[]^ab')
assert(strset('%Z') == strset('[\1-\255]'))
assert(strset('.') == strset('[\1-\255%z]'))
print('+');

assert(strung.match("alo xyzK", "(%w+)K") == "xyz")
assert(strung.match("254 K", "(%d*)K") == "")
assert(strung.match("alo ", "(%w*)$") == "")
assert(strung.match("alo ", "(%w+)$") == nil)
assert(strung.find("(?lo)", "%(?") == 1)
local a, b, c, d, e = strung.match("?lo alo", "^(((.).).* (%w*))$")
assert(a == '?lo alo' and b == '?l' and c == '?' and d == 'alo' and e == nil)
a, b, c, d  = strung.match('0123456789', '(.+(.?)())')
assert(a == '0123456789' and b == '' and c == 11 and d == nil)
print('+')

assert(strung.gsub('?lo ?lo', '?', 'x') == 'xlo xlo')
assert(strung.gsub('alo ?lo  ', ' +$', '') == 'alo ?lo')  -- trim
-- assert(strung.gsub('  alo alo  ', '^%s*(.-)%s*$', '%1') == 'alo alo')  -- double trim<commit_msg>remove debug print statements used to locate the segfault<commit_after>strung = require"strung"

print('testing pattern matching')

function f(s, p)
  local i,e = strung.find(s, p)
  if i then return string.sub(s, i, e) end
end

function f1(s, p)
  p = strung.gsub(p, "%%([0-9])", function (s) return "%" .. (s+1) end)
  p = strung.gsub(p, "^(^?)", "%1()", 1)
  p = strung.gsub(p, "($?)$", "()%1", 1)
  local t = {strung.match(s, p)}
  return strung.sub(s, t[1], t[#t] - 1)
end

a,b = strung.find('', '')    -- empty patterns are tricky
assert(a == 1 and b == 0);
a,b = strung.find('alo', '')
assert(a == 1 and b == 0)
a,b = strung.find('a\0o a\0o a\0o', 'a', 1)   -- first position
assert(a == 1 and b == 1)
a,b = strung.find('a\0o a\0o a\0o', 'a\0o', 2)   -- starts in the midle
assert(a == 5 and b == 7)
a,b = strung.find('a\0o a\0o a\0o', 'a\0o', 9)   -- starts in the midle
assert(a == 9 and b == 11)
a,b = strung.find('a\0a\0a\0a\0\0ab', '\0ab', 2);  -- finds at the end
assert(a == 9 and b == 11);
a,b = strung.find('a\0a\0a\0a\0\0ab', 'b')    -- last position
assert(a == 11 and b == 11)
assert(strung.find('a\0a\0a\0a\0\0ab', 'b\0') == nil)   -- check ending
assert(strung.find('', '\0') == nil)
assert(strung.find('alo123alo', '12') == 4)
assert(strung.find('alo123alo', '^12') == nil)

assert(f('aloALO', '%l*') == 'alo')
assert(f('aLo_ALO', '%a*') == 'aLo')

assert(f('aaab', 'a*') == 'aaa');
assert(f('aaa', '^.*$') == 'aaa');
assert(f('aaa', 'b*') == '');
assert(f('aaa', 'ab*a') == 'aa')
assert(f('aba', 'ab*a') == 'aba')
assert(f('aaab', 'a+') == 'aaa')
assert(f('aaa', '^.+$') == 'aaa')
assert(f('aaa', 'b+') == nil)
assert(f('aaa', 'ab+a') == nil)
assert(f('aba', 'ab+a') == 'aba')
assert(f('a$a', '.$') == 'a')
assert(f('a$a', '.%$') == 'a$')
assert(f('a$a', '.$.') == 'a$a')
assert(f('a$a', '$$') == nil)
assert(f('a$b', 'a$') == nil)
assert(f('a$a', '$') == '')
assert(f('', 'b*') == '')
assert(f('aaa', 'bb*') == nil)
assert(f('aaab', 'a-') == '')
assert(f('aaa', '^.-$') == 'aaa')
assert(f('aabaaabaaabaaaba', 'b.*b') == 'baaabaaabaaab')
assert(f('aabaaabaaabaaaba', 'b.-b') == 'baaab')
assert(f('alo xo', '.o$') == 'xo')
assert(f(' \n isto ? assim', '%S%S*') == 'isto')
assert(f(' \n isto ? assim', '%S*$') == 'assim')
assert(f(' \n isto ? assim', '[a-z]*$') == 'assim')
assert(f('um caracter ? extra', '[^%sa-z]') == '?')
assert(f('', 'a?') == '')
assert(f('?', '??') == '?')
assert(f('?bl', '??b?l?') == '?bl')
assert(f('  ?bl', '??b?l?') == '')
assert(f('aa', '^aa?a?a') == 'aa')
assert(f(']]]?b', '[^]]') == '?')
assert(f("0alo alo", "%x*") == "0a")
assert(f("alo alo", "%C+") == "alo alo")
print('+')

-- assert(f1('alo alx 123 b\0o b\0o', '(..*) %1') == "b\0o b\0o")
-- assert(f1('axz123= 4= 4 34', '(.+)=(.*)=%2 %1') == '3= 4= 4 3')
-- assert(f1('=======', '^(=*)=%1$') == '=======')
assert(strung.match('==========', '^([=]*)=%1$') == nil)

local function range (i, j)
  if i <= j then
    return i, range(i+1, j)
  end
end

local abc = string.char(range(0, 255));

assert(string.len(abc) == 256)

local function strset (p)
  local res = {s=''}
  strung.gsub(abc, p, function (c)
    res.s = res.s .. c
  end)
  return res.s
end;

assert(string.len(strset('[\110-\120]')) == 11)
assert(string.len(strset('[\200-\210]')) == 11)

assert(strset('[a-z]') == "abcdefghijklmnopqrstuvwxyz")
assert(strset('[a-z%d]') == strset('[%da-uu-z]'))
assert(strset('[a-]') == "-a")
assert(strset('[^%W]') == strset('[%w]'))
assert(strset('[]%%]') == '%]')
assert(strset('[a%-z]') == '-az')
assert(strset('[%^%[%-a%]%-b]') == '-[]^ab')
assert(strset('%Z') == strset('[\1-\255]'))
assert(strset('.') == strset('[\1-\255%z]'))
print('+');

assert(strung.match("alo xyzK", "(%w+)K") == "xyz")
assert(strung.match("254 K", "(%d*)K") == "")
assert(strung.match("alo ", "(%w*)$") == "")
assert(strung.match("alo ", "(%w+)$") == nil)
assert(strung.find("(?lo)", "%(?") == 1)
local a, b, c, d, e = strung.match("?lo alo", "^(((.).).* (%w*))$")
assert(a == '?lo alo' and b == '?l' and c == '?' and d == 'alo' and e == nil)
a, b, c, d  = strung.match('0123456789', '(.+(.?)())')
assert(a == '0123456789' and b == '' and c == 11 and d == nil)
print('+')

assert(strung.gsub('?lo ?lo', '?', 'x') == 'xlo xlo')
assert(strung.gsub('alo ?lo  ', ' +$', '') == 'alo ?lo')  -- trim
-- assert(strung.gsub('  alo alo  ', '^%s*(.-)%s*$', '%1') == 'alo alo')  -- double trim
{'fraction_non_alphanumeric': 0.3530190970305773, 'fraction_numerical': 0.03521359973507009, 'mean_word_length': 4.194954128440367, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4757315', 'n_tokens_mistral': 4439, 'n_tokens_neox': 4185, 'n_words': 856}