<commit_before>(*
Zipperposition: a functional superposition prover for prototyping
Copyright (C) 2012 <NAME>

This is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301 USA.
*)

(* {1 Basic S-expressions, with printing and parsing} *)

(** S-expression *)
type t =
  | Atom of string  (** An atom *)
  | List of t list  (** A list of S-expressions *)

(** Token that compose a Sexpr once serialized *)
type token = [`Open | `Close | `Atom of string]

(** Iterate on the S-expression, calling the callback with tokens *)
let rec iter f s = match s with
  | Atom a -> f (`Atom a)
  | List l -> f `Open; iter_list f l; f `Close
and iter_list f l = match l with
  | [] -> ()
  | x::l' -> iter f x; iter_list f l'

(** Traverse. This yields a sequence of tokens *)
let traverse s = Sequence.from_iter (fun k -> iter k s)

(** Lex: create a sequence of tokens from the given in_channel. *)
let lex input =
  let seq_fun k =
    let in_word = ref false in
    let buf = Buffer.create 128 in
    (* loop. TODO handle escaping of () *)
    let rec next () =
      (match input_char input with
      | '(' -> k `Open
      | ')' -> k `Close
      | ' ' | '\t' | '\n' ->
        if !in_word then begin
          (* this whitespace follows a word *)
          let word = Buffer.contents buf in
          Buffer.clear buf;
          in_word := false;
          k (`Atom word)
        end
      | c -> in_word := true; Buffer.add_char buf c);
      next ()
    in
    try next (); (* should raise an exception *)
    with End_of_file -> ()
  in 
  Sequence.from_iter seq_fun

(** Build a Sexpr from a sequence of tokens *)
let of_seq seq =
  let stack = ref [] in
  (* called on every token *)
  let rec k token = match token with
    | `Open -> stack := `Open :: !stack
    | `Close -> stack := collapse [] !stack
    | `Atom a -> stack := (`Expr (Atom a)) :: !stack
  (* collapse last list into an `Expr *)
  and collapse acc stack = match stack with
  | `Open::stack' -> `Expr (List acc) :: stack'
  | `Expr a::stack' -> collapse (a :: acc) stack'
  | _ -> assert false
  in
  (* iterate on the sequence *)
  Sequence.iter k seq;
  (* stack should contain exactly one expression *)
  match !stack with
  | [`Expr expr] -> expr
  | [] -> failwith "no Sexpr could be parsed"
  | _ -> failwith "too many elements on the stack"

(** Print a token on the given formatter *)
let pp_token formatter token = match token with
  | `Open -> Format.fprintf formatter "@[("
  | `Close -> Format.fprintf formatter ")@]"
  | `Atom s -> Format.pp_print_string formatter s
<commit_msg>apparently correct lexing and parsing<commit_after>(*
Zipperposition: a functional superposition prover for prototyping
Copyright (C) 2012 <NAME>

This is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301 USA.
*)

(* {1 Basic S-expressions, with printing and parsing} *)

(** S-expression *)
type t =
  | Atom of string  (** An atom *)
  | List of t list  (** A list of S-expressions *)

(** Token that compose a Sexpr once serialized *)
type token = [`Open | `Close | `Atom of string]

(** Iterate on the S-expression, calling the callback with tokens *)
let rec iter f s = match s with
  | Atom a -> f (`Atom a)
  | List l -> f `Open; iter_list f l; f `Close
and iter_list f l = match l with
  | [] -> ()
  | x::l' -> iter f x; iter_list f l'

(** Traverse. This yields a sequence of tokens *)
let traverse s = Sequence.from_iter (fun k -> iter k s)

(** Lex: create a sequence of tokens from the given in_channel. *)
let lex input =
  let seq_fun k =
    let in_word = ref false in
    let buf = Buffer.create 128 in
    (* loop. TODO handle escaping of () *)
    let rec next () =
      (match input_char input with
      | '(' -> k `Open
      | ')' -> flush_word(); k `Close
      | ' ' | '\t' | '\n' -> flush_word ()
      | c -> in_word := true; Buffer.add_char buf c);
      next ()
    (* finish the previous word token *)
    and flush_word () =
      if !in_word then begin
        (* this whitespace follows a word *)
        let word = Buffer.contents buf in
        Buffer.clear buf;
        in_word := false;
        k (`Atom word)
      end
    in
    try next (); (* should raise an exception *)
    with End_of_file -> ()
  in 
  Sequence.from_iter seq_fun

(** Build a Sexpr from a sequence of tokens *)
let of_seq seq =
  (* called on every token *)
  let rec k stack token = match token with
    | `Open -> `Open :: stack
    | `Close -> collapse [] stack
    | `Atom a -> (`Expr (Atom a)) :: stack
  (* collapse last list into an `Expr *)
  and collapse acc stack = match stack with
  | `Open::stack' -> `Expr (List acc) :: stack'
  | `Expr a::stack' -> collapse (a :: acc) stack'
  | _ -> assert false
  in
  (* iterate on the sequence, given an empty initial stack *)
  let stack = Sequence.fold k [] seq in
  (* stack should contain exactly one expression *)
  match stack with
  | [`Expr expr] -> expr
  | [] -> failwith "no Sexpr could be parsed"
  | _ -> failwith "too many elements on the stack"

(** Print a token on the given formatter *)
let pp_token formatter token = match token with
  | `Open -> Format.fprintf formatter "@[("
  | `Close -> Format.fprintf formatter ")@]"
  | `Atom s -> Format.pp_print_string formatter s

{'fraction_non_alphanumeric': 0.11002025868786038, 'fraction_numerical': 0.006233442418575658, 'mean_word_length': 2.899149453219927, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1399543', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1868, 'n_words': 877}