<commit_before>lpeg = require("lpeg")
local R = lpeg.R
local S = lpeg.S
local P = lpeg.P
local C = lpeg.C
local V = lpeg.V

local number = {}

local digit = R("09")
number.integer = (S("+-") ^ -1) * (digit   ^  1)
number.fractional = (P(".")   ) * (digit ^ 1)
number.decimal =	
	(number.integer *              -- Integer
	(number.fractional ^ -1)) +    -- Fractional
	(S("+-") * number.fractional)  -- Completely fractional number

number.scientific = 
	number.decimal * -- Decimal number
	S("Ee") *        -- E or e
	number.integer   -- Exponent

-- Matches all of the above
number.number = C(number.decimal + number.scientific) / function (n) return tonumber(n) end
local identifier = (R("AZ") + R("az") + P("_") + R("09"))^1
local whitespace = S('\r\n\f\t ')^0
local units = P("mm") + P("cm") + P("in") + P("pt")
local dimensioned_string = ( C(number.number) * whitespace * C(units) ) / function (x,n,u) return  SILE.toPoints(n, u) end
local func = C(P("top") + P("left") + P("bottom") + P("right") ) * P("(") * C(identifier) * P(")") / function (dim, ident) f = SILE.getFrame(ident); return f[dim](f) end
local percentage = ( C(number.number) * whitespace * P("%") ) / function (n) return SILE.toPoints(n, "%", SILE.documentState._dimension) end
local primary = dimensioned_string + percentage + func + number.number<commit_msg>Add a function for parsing glue definitions. (Rethink if it should be in the frame parser or if we should split to multiple parsers.)<commit_after>lpeg = require("lpeg")
local R = lpeg.R
local S = lpeg.S
local P = lpeg.P
local C = lpeg.C
local V = lpeg.V
local Cg = lpeg.Cg
local Ct = lpeg.Ct
local number = {}

local digit = R("09")
number.integer = (S("+-") ^ -1) * (digit   ^  1)
number.fractional = (P(".")   ) * (digit ^ 1)
number.decimal =	
	(number.integer *              -- Integer
	(number.fractional ^ -1)) +    -- Fractional
	(S("+-") * number.fractional)  -- Completely fractional number

number.scientific = 
	number.decimal * -- Decimal number
	S("Ee") *        -- E or e
	number.integer   -- Exponent

-- Matches all of the above
number.number = C(number.decimal + number.scientific) / function (n) return tonumber(n) end
local identifier = (R("AZ") + R("az") + P("_") + R("09"))^1
local whitespace = S('\r\n\f\t ')^0
local units = P("mm") + P("cm") + P("in") + P("pt")
local dimensioned_string = ( C(number.number) * whitespace * C(units) ) / function (x,n,u) return  SILE.toPoints(n, u) end
local func = C(P("top") + P("left") + P("bottom") + P("right") ) * P("(") * C(identifier) * P(")") / function (dim, ident) f = SILE.getFrame(ident); return f[dim](f) end
local percentage = ( C(number.number) * whitespace * P("%") ) / function (n) return SILE.toPoints(n, "%", SILE.documentState._dimension) end
local primary = dimensioned_string + percentage + func + number.number
local zero = P("0") / function(...) return 0 end
SILE.nodefactory.glueParser = Ct(Cg(dimensioned_string + zero, "length") * whitespace * (P("plus") * whitespace * Cg(dimensioned_string + zero, "stretch"))^-1 * whitespace * (P("minus") * whitespace * Cg(dimensioned_string + zero,"shrink"))^-1)
{'fraction_non_alphanumeric': 0.19197431781701443, 'fraction_numerical': 0.0077046548956661316, 'mean_word_length': 3.699849170437406, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095682', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1104, 'n_words': 302}