<commit_before>-- Copyright 2017 <NAME>. See LICENSE.
-- Standard ML LPeg lexer.

local l = require('lexer')
local token = l.token

function mlword(words)
  return l.word_match(words, "'")
end

local ws = token(l.WHITESPACE, l.space^1)

-- single line comments are valid in successor ml
local cl = '(*)' * l.nonnewline^0
local comment = token(l.COMMENT, cl + l.nested_pair('(*', '*)'))

local string = token(l.STRING, lpeg.P('#')^-1 * l.delimited_range('"', true))

local function num(digit)
  return digit * (digit^0 * lpeg.P('_'))^0 * digit^1 + digit
end

local int = num(l.digit)
local frac = lpeg.P('.') * int
local minus = lpeg.P('~')^-1
local exp = lpeg.S('eE') * minus * int
local real = int * frac^-1 * exp + int * frac * exp^-1
local hex = num(l.xdigit)
local bin = num(lpeg.S('01'))

local number = token(l.NUMBER,
  lpeg.P('0w') * int
  + (lpeg.P('0wx') + lpeg.P('0xw')) * hex
  + (lpeg.P('0wb') + lpeg.P('0bw')) * bin
  + minus * lpeg.P('0x') * hex
  + minus * lpeg.P('0b') * bin
  + minus * real
  + minus * int
)

local keyword = token(l.KEYWORD, mlword{
  'abstype', 'and', 'andalso', 'as', 'case', 'do', 'datatype', 'else', 'end',
  'exception', 'fn', 'fun', 'handle', 'if', 'in', 'infix', 'infixr', 'let',
  'local', 'nonfix', 'of', 'op', 'orelse', 'raise', 'rec', 'then',
  'type', 'val', 'with', 'withtype', 'while',

  'eqtype', 'functor', 'include', 'sharing', 'sig', 'signature',
  'struct', 'structure'
})

-- includes valid symbols for identifiers
local operator = token(l.OPERATOR, lpeg.S('!*/+-^:@=<>()[]{},;._|#%&$?~`\\'))

local type = token(l.TYPE, mlword{
  'int', 'real', 'word', 'bool', 'char', 'string', 'unit',
  'array', 'exn', 'list', 'option', 'order', 'ref', 'substring', 'vector'
})

-- `real`, `vector` and `substring` are a problem
local func = token(l.FUNCTION, mlword{
  'app', 'before', 'ceil', 'chr', 'concat', 'exnMessage', 'exnName',
  'explode', 'floor', 'foldl', 'foldr', 'getOpt', 'hd', 'ignore',
  'implode', 'isSome', 'length', 'map', 'not', 'null', 'ord', 'print',
  'real', 'rev', 'round', 'size', 'str', 'substring', 'tl', 'trunc',
  'valOf', 'vector',
  'o', 'abs', 'mod', 'div'
})

-- non-symbolic identifiers only
local id = (l.alnum + "'" + '_')^0
local aid = l.alpha * id
local longid = (aid * lpeg.P('.'))^0 * aid
local identifier = token(l.IDENTIFIER, l.lower * id)
local typevar = token(l.VARIABLE, "'" * id)
local c = mlword{'true', 'false', 'nil'}
local const = token(l.CONSTANT, l.upper * id + c)
local structure = token(l.CLASS, aid * lpeg.P('.'))

local open
  = token(l.KEYWORD, mlword{'open', 'structure', 'functor'})
  * ws * token(l.CLASS, longid)

local struct_dec
  = token(l.KEYWORD, lpeg.P('structure')) * ws
  * token(l.CLASS, aid) * ws
  * token(l.OPERATOR, lpeg.P('=')) * ws

local struct_new = struct_dec * token(l.KEYWORD, lpeg.P('struct'))
local struct_alias = struct_dec * token(l.CLASS, longid)
<commit_msg>Began refactoring SML lexer to be more object-oriented.<commit_after>-- Copyright 2017 <NAME>. See LICENSE.
-- Standard ML LPeg lexer.

local lexer = require('lexer')
local token = lexer.token

function mlword(words)
  return lexer.word_match(words, "'")
end

local ws = token(lexer.WHITESPACE, lexer.space^1)

-- single line comments are valid in successor ml
local cl = '(*)' * lexer.nonnewline^0
local comment = token(lexer.COMMENT, cl + lexer.nested_pair('(*', '*)'))

local string = token(lexer.STRING, lpeg.P('#')^-1 *
                                   lexer.delimited_range('"', true))

local function num(digit)
  return digit * (digit^0 * lpeg.P('_'))^0 * digit^1 + digit
end

local int = num(lexer.digit)
local frac = lpeg.P('.') * int
local minus = lpeg.P('~')^-1
local exp = lpeg.S('eE') * minus * int
local real = int * frac^-1 * exp + int * frac * exp^-1
local hex = num(lexer.xdigit)
local bin = num(lpeg.S('01'))

local number = token(lexer.NUMBER,
  lpeg.P('0w') * int
  + (lpeg.P('0wx') + lpeg.P('0xw')) * hex
  + (lpeg.P('0wb') + lpeg.P('0bw')) * bin
  + minus * lpeg.P('0x') * hex
  + minus * lpeg.P('0b') * bin
  + minus * real
  + minus * int
)

local keyword = token(lexer.KEYWORD, mlword{
  'abstype', 'and', 'andalso', 'as', 'case', 'do', 'datatype', 'else', 'end',
  'exception', 'fn', 'fun', 'handle', 'if', 'in', 'infix', 'infixr', 'let',
  'local', 'nonfix', 'of', 'op', 'orelse', 'raise', 'rec', 'then',
  'type', 'val', 'with', 'withtype', 'while',

  'eqtype', 'functor', 'include', 'sharing', 'sig', 'signature',
  'struct', 'structure'
})

-- includes valid symbols for identifiers
local operator = token(lexer.OPERATOR,
                       lpeg.S('!*/+-^:@=<>()[]{},;._|#%&$?~`\\'))

local type = token(lexer.TYPE, mlword{
  'int', 'real', 'word', 'bool', 'char', 'string', 'unit',
  'array', 'exn', 'list', 'option', 'order', 'ref', 'substring', 'vector'
})

-- `real`, `vector` and `substring` are a problem
local func = token(lexer.FUNCTION, mlword{
  'app', 'before', 'ceil', 'chr', 'concat', 'exnMessage', 'exnName',
  'explode', 'floor', 'foldl', 'foldr', 'getOpt', 'hd', 'ignore',
  'implode', 'isSome', 'length', 'map', 'not', 'null', 'ord', 'print',
  'real', 'rev', 'round', 'size', 'str', 'substring', 'tl', 'trunc',
  'valOf', 'vector',
  'o', 'abs', 'mod', 'div'
})

-- non-symbolic identifiers only
local id = (lexer.alnum + "'" + '_')^0
local aid = lexer.alpha * id
local longid = (aid * lpeg.P('.'))^0 * aid
local identifier = token(lexer.IDENTIFIER, lexer.lower * id)
local typevar = token(lexer.VARIABLE, "'" * id)
local c = mlword{'true', 'false', 'nil'}
local const = token(lexer.CONSTANT, lexer.upper * id + c)
local structure = token(lexer.CLASS, aid * lpeg.P('.'))

local open
  = token(lexer.KEYWORD, mlword{'open', 'structure', 'functor'})
  * ws * token(lexer.CLASS, longid)

local struct_dec
  = token(lexer.KEYWORD, lpeg.P('structure')) * ws
  * token(lexer.CLASS, aid) * ws
  * token(lexer.OPERATOR, lpeg.P('=')) * ws

local struct_new = struct_dec * token(lexer.KEYWORD, lpeg.P('struct'))
local struct_alias = struct_dec * token(lexer.CLASS, longid)

{'fraction_non_alphanumeric': 0.23194675540765392, 'fraction_numerical': 0.007986688851913478, 'mean_word_length': 4.217881944444445, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249366', 'n_tokens_mistral': 2394, 'n_tokens_neox': 2321, 'n_words': 599}