<commit_before>open Syntax

let parse_label = function
  | [(("label"|"LABEL"), l)] -> l
  | _ -> assert false

let parse_expr s :expr =
  let open MParser in
  let app op x y = Op (op, x, y) in
  let infix sym f assoc = Infix  (Tokens.skip_symbol sym >> return (app f), assoc) in
  let prefix sym f      = Prefix (Tokens.skip_symbol sym >> return f) in
  let operators:(('a, 's) operator list) list =
    [ [ prefix "-" (fun x -> Op (Sub, Num 0., x)) ]
    ; [ infix "*" Mul Assoc_left
      ; infix "/" Div Assoc_left
      ; infix "%" Mod Assoc_left
      ]
    ; [ infix "+" Add Assoc_left
      ; infix "-" Sub Assoc_left
      ]
    ] in
  let number =
    let num x = Num x in
    regexp (make_regexp "-?\\d+(\\.\\d*)?\\s*") >>= fun s ->
    s |> String.trim |> float_of_string |> num |> return
  in
  let rand = string "$rand" >> spaces >> return Rand in
  let rank = string "$rank" >> spaces >> return Rank in
  let param =
    char '$' >>
    Tokens.decimal >>= fun n -><commit_msg>fix build with recent mparser<commit_after>open Syntax

let parse_label = function
  | [(("label"|"LABEL"), l)] -> l
  | _ -> assert false

let parse_expr s :expr =
  let open MParser in
  let open MParser_RE in
  let app op x y = Op (op, x, y) in
  let infix sym f assoc = Infix  (Tokens.skip_symbol sym >> return (app f), assoc) in
  let prefix sym f      = Prefix (Tokens.skip_symbol sym >> return f) in
  let operators:(('a, 's) operator list) list =
    [ [ prefix "-" (fun x -> Op (Sub, Num 0., x)) ]
    ; [ infix "*" Mul Assoc_left
      ; infix "/" Div Assoc_left
      ; infix "%" Mod Assoc_left
      ]
    ; [ infix "+" Add Assoc_left
      ; infix "-" Sub Assoc_left
      ]
    ] in
  let number =
    let num x = Num x in
    regexp (make_regexp "-?\\d+(\\.\\d*)?\\s*") >>= fun s ->
    s |> String.trim |> float_of_string |> num |> return
  in
  let rand = string "$rand" >> spaces >> return Rand in
  let rank = string "$rank" >> spaces >> return Rank in
  let param =
    char '$' >>
    Tokens.decimal >>= fun n ->
{'fraction_non_alphanumeric': 0.15604938271604937, 'fraction_numerical': 0.0009876543209876543, 'mean_word_length': 2.382303839732888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4508693', 'n_tokens_mistral': 735, 'n_tokens_neox': 730, 'n_words': 252}