<commit_before>open Core.Std

module Op = struct

  type t =
    | Nop
    | Indent
    | Dedent
    | Comment of string
    | Text of string
    | Space of int
    | Newline

  let write buf ops =
    let open Buffer in
    ignore @@ List.fold_left ~init:Nop ops ~f:(fun prev op ->
        begin match op with
          | Text s -> add_string buf s
          | Space n ->
            for i = 0 to n-1 do
              add_string buf " "
            done
          | Newline -> add_string buf "\n"
          | _ -> ()
        end;
        op)

end

module Context = struct

  type t = {
    buf : Buffer.t;
    mutable ops : Op.t list;
    mutable indent_lv : int;
  }

  let create buf =
    { buf; ops = []; indent_lv = 0 }

  let contents ctx =
    List.rev ctx.ops

  let add ctx op =
    ctx.ops <- op :: ctx.ops

  let text ctx s =
    add ctx @@ Op.Text s

  let word ctx s =
    text ctx s

  let rarrow ctx =
    add ctx @@ Op.Text "->"

  let space ctx =
    add ctx @@ Op.Space 1

  let spaces ctx n =
    add ctx @@ Op.Space n

  let newline ctx =
    add ctx @@ Op.Newline

  let newlines ?(n=2) ctx =
    for i = 1 to n do
      add ctx @@ Op.Newline
    done

  let dot_newline ctx =
    text ctx ".";
    newline ctx

  let dot_newlines ?(n=2) ctx =
    text ctx ".";
    newlines ctx ~n

  let indent ctx =
    ctx.indent_lv <- ctx.indent_lv + 1;
    newline ctx

  let indent_spaces ctx =
    spaces ctx @@ ctx.indent_lv * 4

  let dedent ctx =
    ctx.indent_lv <- ctx.indent_lv - 1;
    newline ctx

end

let rec write ctx node =
  let open Ast_intf in
  let open Context in
  let open Located in

  let iter nodes =
    List.iter nodes ~f:(fun node -> write ctx node)
  in

  let write_sep opt sep =
    match opt with
    | None -> ()
    | Some _ -> text ctx sep
  in

  let write_seplist seplist sep =
    Seplist.iter seplist ~f:(fun sep_opt node ->
        write ctx node;
        write_sep sep_opt sep)
  in

  let write_when_guard w guard =
    match (w, guard) with
    | Some w, Some guard ->
      text ctx "when ";
      write_seplist guard ", ";
      text ctx " "
    | _ -> ()
  in

  let write_fun_sig fsig =
    text ctx fsig.fun_sig_name.desc;
    text ctx "/";
    text ctx fsig.fun_sig_arity.desc
  in

  let write_fun_sigs fsigs =
    Seplist.iter fsigs
      ~f:(fun sep fsig ->
          write_fun_sig fsig;
          write_sep sep ", ")
  in

  let write_pattern ptn =
    write ctx ptn
  in
<commit_msg>format exp-list<commit_after>open Core.Std

module Op = struct

  type t =
    | Nop
    | Indent
    | Dedent
    | Comment of string
    | Text of string
    | Space of int
    | Newline

  let write buf ops =
    let open Buffer in
    ignore @@ List.fold_left ~init:Nop ops ~f:(fun prev op ->
        begin match op with
          | Text s -> add_string buf s
          | Space n ->
            for i = 0 to n-1 do
              add_string buf " "
            done
          | Newline -> add_string buf "\n"
          | _ -> ()
        end;
        op)

end

module Context = struct

  type t = {
    buf : Buffer.t;
    mutable ops : Op.t list;
    mutable indent_lv : int;
  }

  let create buf =
    { buf; ops = []; indent_lv = 0 }

  let contents ctx =
    List.rev ctx.ops

  let add ctx op =
    ctx.ops <- op :: ctx.ops

  let text ctx s =
    add ctx @@ Op.Text s

  let word ctx s =
    text ctx s

  let rarrow ctx =
    add ctx @@ Op.Text "->"

  let space ctx =
    add ctx @@ Op.Space 1

  let spaces ctx n =
    add ctx @@ Op.Space n

  let newline ctx =
    add ctx @@ Op.Newline

  let newlines ?(n=2) ctx =
    for i = 1 to n do
      add ctx @@ Op.Newline
    done

  let dot_newline ctx =
    text ctx ".";
    newline ctx

  let dot_newlines ?(n=2) ctx =
    text ctx ".";
    newlines ctx ~n

  let indent ctx =
    ctx.indent_lv <- ctx.indent_lv + 1;
    newline ctx

  let indent_spaces ctx =
    spaces ctx @@ ctx.indent_lv * 4

  let dedent ctx =
    ctx.indent_lv <- ctx.indent_lv - 1;
    newline ctx

end

let rec write ctx node =
  let open Ast_intf in
  let open Context in
  let open Located in

  let iter nodes =
    List.iter nodes ~f:(fun node -> write ctx node)
  in

  let write_sep opt sep =
    match opt with
    | None -> ()
    | Some _ -> text ctx sep
  in

  let write_seplist seplist sep =
    Seplist.iter seplist ~f:(fun sep_opt node ->
        write ctx node;
        write_sep sep_opt sep)
  in

  let write_exp_list exp_list =
    write_seplist exp_list ", "
  in

  let write_when_guard w guard =
    match (w, guard) with
    | Some w, Some guard ->
      text ctx "when ";
      write_seplist guard ", ";
      text ctx " "
    | _ -> ()
  in

  let write_fun_sig fsig =
    text ctx fsig.fun_sig_name.desc;
    text ctx "/";
    text ctx fsig.fun_sig_arity.desc
  in

  let write_fun_sigs fsigs =
    Seplist.iter fsigs
      ~f:(fun sep fsig ->
          write_fun_sig fsig;
          write_sep sep ", ")
  in

  let write_pattern ptn =
    write ctx ptn
  in

{'fraction_non_alphanumeric': 0.08876811594202899, 'fraction_numerical': 0.004025764895330112, 'mean_word_length': 1.731720725673447, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1008556', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1693, 'n_words': 607}