<commit_before>-- Use of this source code is governed by the Apache 2.0 license; see COPYING.

module(..., package.seeall)
local maxpc = require("lib.maxpc")
local match, capture, combine = maxpc.import()
local codepoint = maxpc.codepoint

-- Implementation of regular expressions as defined in Appendix G of "W3C XML
-- Schema Definition Language (XSD) 1.1 Part 2: Datatypes", see:
--
--    https://www.w3.org/TR/xmlschema11-2/#regexs
--
-- The main entry function `regexp.compile' accepts a regular expression
-- string, and returns a predicate function that tests whether a string is part
-- of the language defined by the expression.
--
-- Example:
--    local is_identifier = regexp.compile("[a-zA-Z][a-zA-Z0-9]*")
--    is_identifier("Foo3") -> true
--    is_identifier("7up") -> false
--
-- It uses a combinatory parsing library (MaxPC) to parse a regular expression
-- in the format defined by the specification referenced above, and compiles
-- the denoted regular language to a MaxPC grammar.
--
-- NYI: Block escapes and Unicode support for category escapes are not
-- implemented. Category escapes and complements only match codepoints in the
-- Basic Latin block (ASCII). Users of category escapes and complements need to
-- ensure their input is ASCII-only.

function compile (expr)
   local ast = parse(expr)
   local parser = compile_branches(ast.branches)
   return function (str)
      local _, success, is_eof = maxpc.parse(str, parser)
      return success and is_eof
   end
end

local regExp_parser -- forward definition

function parse (expr)
   local result, success, is_eof = maxpc.parse(expr, regExp_parser)
   if not (success and is_eof) then
      error("Unable to parse regular expression: " .. expr)
   else
      return result
   end
end


-- Parser rules: string -> AST

function capture.regExp ()
   return capture.unpack(
      capture.seq(capture.branch(), combine.any(capture.otherBranch())),
      function (branch, otherBranches)
         local branches = {branch}
         for _, branch in ipairs(otherBranches or {}) do
            table.insert(branches, branch)
         end
         return {branches=branches}
      end
   )
end

function capture.branch ()
   return capture.transform(combine.any(capture.piece()),
                            function (pieces) return {pieces=pieces} end)
end

function capture.otherBranch ()
   return capture.unpack(
      capture.seq(match.equal("|"), capture.branch()),
      function (_, branch) return branch end
   )
end

function capture.piece ()
   return capture.unpack(
      capture.seq(capture.atom(), combine.maybe(capture.quantifier())),
      function (atom, quantifier)
         return {atom=atom, quantifier=quantifier or nil}
      end
   )
end

function capture.quantifier ()
   return combine._or(
      capture.subseq(match.equal("?")),
      capture.subseq(match.equal("*")),
      capture.subseq(match.equal("+")),
      capture.unpack(
         capture.seq(match.equal("{"), capture.quantity(), match.equal("}")),
         function (_, quantity, _) return quantity end
      )
   )
end

function match.digit (s)
   return match.satisfies(function (s) return member(s, "0123456789") end)
end

function capture.quantity ()
   return combine._or(
      capture.quantRange(),
      capture.quantMin(),
      capture.transform(capture.quantExact(),
                        function (n) return {exactly=n} end)
   )
end

function capture.quantRange ()
   return capture.unpack(
      capture.seq(capture.quantExact(),
                  match.equal(","),
                  capture.quantExact()),
      function (min, _, max) return {min=min, max=max} end<commit_msg>lib.xsd_regexp: remove obsolete match.digit<commit_after>-- Use of this source code is governed by the Apache 2.0 license; see COPYING.

module(..., package.seeall)
local maxpc = require("lib.maxpc")
local match, capture, combine = maxpc.import()
local codepoint = maxpc.codepoint

-- Implementation of regular expressions as defined in Appendix G of "W3C XML
-- Schema Definition Language (XSD) 1.1 Part 2: Datatypes", see:
--
--    https://www.w3.org/TR/xmlschema11-2/#regexs
--
-- The main entry function `regexp.compile' accepts a regular expression
-- string, and returns a predicate function that tests whether a string is part
-- of the language defined by the expression.
--
-- Example:
--    local is_identifier = regexp.compile("[a-zA-Z][a-zA-Z0-9]*")
--    is_identifier("Foo3") -> true
--    is_identifier("7up") -> false
--
-- It uses a combinatory parsing library (MaxPC) to parse a regular expression
-- in the format defined by the specification referenced above, and compiles
-- the denoted regular language to a MaxPC grammar.
--
-- NYI: Block escapes and Unicode support for category escapes are not
-- implemented. Category escapes and complements only match codepoints in the
-- Basic Latin block (ASCII). Users of category escapes and complements need to
-- ensure their input is ASCII-only.

function compile (expr)
   local ast = parse(expr)
   local parser = compile_branches(ast.branches)
   return function (str)
      local _, success, is_eof = maxpc.parse(str, parser)
      return success and is_eof
   end
end

local regExp_parser -- forward definition

function parse (expr)
   local result, success, is_eof = maxpc.parse(expr, regExp_parser)
   if not (success and is_eof) then
      error("Unable to parse regular expression: " .. expr)
   else
      return result
   end
end


-- Parser rules: string -> AST

function capture.regExp ()
   return capture.unpack(
      capture.seq(capture.branch(), combine.any(capture.otherBranch())),
      function (branch, otherBranches)
         local branches = {branch}
         for _, branch in ipairs(otherBranches or {}) do
            table.insert(branches, branch)
         end
         return {branches=branches}
      end
   )
end

function capture.branch ()
   return capture.transform(combine.any(capture.piece()),
                            function (pieces) return {pieces=pieces} end)
end

function capture.otherBranch ()
   return capture.unpack(
      capture.seq(match.equal("|"), capture.branch()),
      function (_, branch) return branch end
   )
end

function capture.piece ()
   return capture.unpack(
      capture.seq(capture.atom(), combine.maybe(capture.quantifier())),
      function (atom, quantifier)
         return {atom=atom, quantifier=quantifier or nil}
      end
   )
end

function capture.quantifier ()
   return combine._or(
      capture.subseq(match.equal("?")),
      capture.subseq(match.equal("*")),
      capture.subseq(match.equal("+")),
      capture.unpack(
         capture.seq(match.equal("{"), capture.quantity(), match.equal("}")),
         function (_, quantity, _) return quantity end
      )
   )
end

function capture.quantity ()
   return combine._or(
      capture.quantRange(),
      capture.quantMin(),
      capture.transform(capture.quantExact(),
                        function (n) return {exactly=n} end)
   )
end

function capture.quantRange ()
   return capture.unpack(
      capture.seq(capture.quantExact(),
                  match.equal(","),
                  capture.quantExact()),
      function (min, _, max) return {min=min, max=max} end
{'fraction_non_alphanumeric': 0.11490125673249552, 'fraction_numerical': 0.005247893937301478, 'mean_word_length': 2.998895637769188, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7211915', 'n_tokens_mistral': 2105, 'n_tokens_neox': 2055, 'n_words': 653}