<commit_before>{
open Parser

module LE = Lexing

let debug = Trace.debug "lexer"

let lexeme = LE.lexeme

type f_info = (string * int * int)
let info_stk = ref []

let filename () = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.filename : info stack is empty."))
  | (fn,_,_)::_ -> fn
      
let lineno () = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.lineno : info stack is empty."))
  | (_,l,_)::_ -> l

let linestart () = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.linestart : info stack is empty."))
  | (_,_,c)::_ -> c

let set_filename fn = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.set_filename : info stack is empty."))
  | (_,l,c)::t -> info_stk := (fn,l,c)::t
      
let set_lineno l = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.set_lineno : info stack is empty."))
  | (fn,_,c)::t -> info_stk := (fn,l,c)::t
      
let set_linestart c = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.set_linestart : info stack is empty."))
  | (fn,l,_)::t -> info_stk := (fn,l,c)::t

let setup fn = info_stk := (fn,1,0)::!info_stk

let finish () = match !info_stk with
    [] -> raise (Error.Fatal_error("Lexer.finish : info stack is empty."))
  | _::t -> info_stk := t
      
let newline lexbuf : unit = 
  set_linestart (LE.lexeme_start lexbuf);
  set_lineno (lineno () + 1)

let info lexbuf : Info.t = 
  let c1 = LE.lexeme_start lexbuf in
  let c2 = LE.lexeme_end lexbuf in
  let l = lineno () in
  let c = linestart () in
    (l, c1 - c),(l, c2 - c)
      
let error lexbuf msg =
  let i = info lexbuf in
  let t = lexeme lexbuf in   
  let s = Printf.sprintf "Lexing error: %s : %s" msg t in
    raise (Error.Compile_error(i, filename (), s))

let text = Lexing.lexeme

let extractLineno yytext offset =
  int_of_string (String.sub yytext offset (String.length yytext - offset))
    
(* dictionary of Focal keywords *)
let keywords = Hashtbl.create 17
let _ = 
  Safelist.iter 
    (fun (kw,tok) -> Hashtbl.add keywords kw tok )
    [ "let", (fun i -> LET i)
    ; "in", (fun i -> IN i)
    ; "fun", (fun i -> FUN i)
    ; "and", (fun i -> AND i)<commit_msg>oops. my change to the lexer vanished. here it is<commit_after>{
open Parser

module LE = Lexing

let debug = Trace.debug "lexer"

let lexeme = LE.lexeme

type f_info = (string * int * int)
let info_stk = ref []

let filename () = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.filename : info stack is empty."))
  | (fn,_,_)::_ -> fn
      
let lineno () = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.lineno : info stack is empty."))
  | (_,l,_)::_ -> l

let linestart () = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.linestart : info stack is empty."))
  | (_,_,c)::_ -> c

let set_filename fn = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.set_filename : info stack is empty."))
  | (_,l,c)::t -> info_stk := (fn,l,c)::t
      
let set_lineno l = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.set_lineno : info stack is empty."))
  | (fn,_,c)::t -> info_stk := (fn,l,c)::t
      
let set_linestart c = match !info_stk with 
    [] -> raise (Error.Fatal_error("Lexer.set_linestart : info stack is empty."))
  | (fn,l,_)::t -> info_stk := (fn,l,c)::t

let setup fn = info_stk := (fn,1,0)::!info_stk

let finish () = match !info_stk with
    [] -> raise (Error.Fatal_error("Lexer.finish : info stack is empty."))
  | _::t -> info_stk := t
      
let newline lexbuf : unit = 
  set_linestart (LE.lexeme_start lexbuf);
  set_lineno (lineno () + 1)

let info lexbuf : Info.t = 
  let c1 = LE.lexeme_start lexbuf in
  let c2 = LE.lexeme_end lexbuf in
  let l = lineno () in
  let c = linestart () in
    (l, c1 - c - 1),(l, c2 - c - 1)
      
let error lexbuf msg =
  let i = info lexbuf in
  let t = lexeme lexbuf in   
  let s = Printf.sprintf "Lexing error: %s : %s" msg t in
    raise (Error.Compile_error(i, filename (), s))

let text = Lexing.lexeme

let extractLineno yytext offset =
  int_of_string (String.sub yytext offset (String.length yytext - offset))
    
(* dictionary of Focal keywords *)
let keywords = Hashtbl.create 17
let _ = 
  Safelist.iter 
    (fun (kw,tok) -> Hashtbl.add keywords kw tok )
    [ "let", (fun i -> LET i)
    ; "in", (fun i -> IN i)
    ; "fun", (fun i -> FUN i)
    ; "and", (fun i -> AND i)
{'fraction_non_alphanumeric': 0.15765561108587006, 'fraction_numerical': 0.004543389368468878, 'mean_word_length': 2.9382826475849733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2678000', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1676, 'n_words': 488}