<commit_before>(*
   Copyright 2009, 2010, 2011, 2012, 2013 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*)

(*
 * This module contain functionality that is used by various parts of
 * piqi compilers.
 *)

module T = Piqic_piqi


module Record = T.Record
module Field = T.Field
module Variant = T.Variant
module Option = T.Option
module Enum = T.Enum
module Alias = T.Alias


module Import = T.Import
module Func = T.Func
module Any = T.Any


module R = Record
module F = Field
module V = Variant
module O = Option
module E = Enum
module A = Alias
module L = T.Piqi_list
module P = T.Piqi


module U = Piqi_util


module Iolist = Piqi_iolist
open Iolist
<commit_msg>Provide Iolist implementation (further minimize dependency on piqilib)<commit_after>(*
   Copyright 2009, 2010, 2011, 2012, 2013 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*)

(*
 * This module contain functionality that is used by various parts of
 * piqi compilers.
 *)

module T = Piqic_piqi


module Record = T.Record
module Field = T.Field
module Variant = T.Variant
module Option = T.Option
module Enum = T.Enum
module Alias = T.Alias


module Import = T.Import
module Func = T.Func
module Any = T.Any


module R = Record
module F = Field
module V = Variant
module O = Option
module E = Enum
module A = Alias
module L = T.Piqi_list
module P = T.Piqi


module U = Piqi_util


(* a datastructure for output construction *)
module Iolist =
  struct
    type iolist =
        Ios of string
      | Iol of iolist list
      | Iob of char
      | Eol

    (* iolist construction *)
    let (^^) a b =
      match a, b with
        | Ios _, Iol b -> Iol (a::b)
        | Ios " ", Eol -> Eol
        | _, _ -> Iol [a;b]

    let eol = Eol
    let ios x = Ios x
    let iol l = Iol l
    let iob b = Iob b

    let iod delim = function  (* iol with elements separated by delim *)
      | [] -> Iol []
      | h::t ->
        let d = ios delim in
        List.fold_left (fun accu x -> accu ^^ d ^^ x) h t

    let ioq x = (* double-quoted string *)
      iol [ios "\""; ios x; ios "\""]

    (* iolist output *)
    let to_buffer0 buf l =
      let rec aux = function
        | Eol -> Buffer.add_char buf '\n';
        | Ios s -> Buffer.add_string buf s
        | Iol l -> List.iter aux l
        | Iob b -> Buffer.add_char buf b
      in aux l

    let to_buffer l =
      let buf = Buffer.create 4096 in
      to_buffer0 buf l;
      buf

    let to_string l =
      let buf = to_buffer l in
      Buffer.contents buf

    let to_channel ch l =
      let buf = to_buffer l in
      Buffer.output_buffer ch buf
  end
open Iolist

{'fraction_non_alphanumeric': 0.07620099392600774, 'fraction_numerical': 0.014908890115958034, 'mean_word_length': 2.5589390962671907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4524643', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1097, 'n_words': 487}