<commit_before>(*
   Copyright 2009, 2010, 2011 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*)

(* Runtime support for piqi/Protocol Buffers wire format encoding
 *
 * Encoding rules follow this specification:
 *
 *   http://code.google.com/apis/protocolbuffers/docs/encoding.html
 *)


(*
 * Runtime support for parsers (decoders).
 *
 *)

exception Error of int * string


let string_of_loc pos =
  string_of_int pos


let strerr loc s = 
  string_of_loc loc ^ ": " ^ s


let buf_error loc s =
  (*
  failwith (strerr s loc)
  *)
  raise (Error (loc, s))


let error obj s =
  let loc = -1 in (* TODO, XXX: obj location db? *)
  buf_error loc s


module IBuf =
  struct
    type string_buf = 
      {
        s : string;
        start_pos : int;
        len :int;
        mutable pos : int; 
      }

    type stream_buf = char Stream.t

    type t =
      | Stream of stream_buf
      | String of string_buf


    let of_channel x =
      Stream (Stream.of_channel x)


    let of_string x start_pos =
      String
        { s = x; len = String.length x;
          start_pos = start_pos; pos = 0; 
        }


    let to_string buf =
      match buf with
        | Stream x ->
            let res = Buffer.create 20 in
            Stream.iter (Buffer.add_char res) x;
            Buffer.contents res
        | String x ->
            String.sub x.s x.pos (x.len - x.pos)


    let pos buf =
      match buf with
        | Stream x -> Stream.count x
        | String x -> x.pos + x.start_pos


    let error buf s =
      let loc = pos buf in
      buf_error loc s


    let next buf =
      try
        match buf with
          | Stream x -> Stream.next x
          | String x ->
              if x.pos >= x.len
              then
                raise Stream.Failure
              else
                let res = x.s.[x.pos] in
                x.pos <- x.pos + 1;
                res
      with Stream.Failure -> 
        error buf "unexpected end of block while reading integer"


    let skip_one buf =
      match buf with
        | Stream x -> Stream.junk x
        | String x -> x.pos <- x.pos + 1


    let is_empty buf =
      match buf with
        | Stream x ->
            Stream.peek x = None
        | String x ->
            x.pos = x.len


    let next_block buf length =
      try
        match buf with
          | Stream x ->
              let s = String.create length in
              let start_pos = Stream.count x in
              for i = 0 to length-1
              do
                s.[i] <- Stream.next x
              done;
              of_string s start_pos
          | String x ->
              if x.pos + length > x.len
              then
                (* XXX: adjusting position to provide proper EOB location *)
                (x.pos <- x.len; raise Stream.Failure)
              else
                (* NOTE: start_pos, pos and the string itself remain the same in
                 * the new buffer *)
                let res = String { x with len = x.pos + length } in
                (* skip the new buffer in the current buffer *)
                x.pos <- x.pos + length;
                res
      with Stream.Failure -> 
        error buf "unexpected end of block"


    let of_string x =
      of_string x 0
  end


type t = 
  | Varint of int
  | Varint64 of int64 (* used if int width is not enough *)
  | Int32 of int32
  | Int64 of int64
  | Block of IBuf.t
  | Top_block of IBuf.t (* top-level block *)


(* initializers for embedded records/variants (i.e. their contents start without
 * any leading headers/delimiters/separators) *)
let init_from_channel ch =
  Top_block (IBuf.of_channel ch)


let init_from_string s =
  Top_block (IBuf.of_string s)


let is_empty buf =
  IBuf.is_empty buf


let error_variant obj code =
  error obj ("unknown variant: " ^ string_of_int code)
let error_missing obj code =
  error obj  ("missing field " ^ string_of_int code)

let error_enum_obj obj = error obj "enum (varint) expected"
let error_enum_const obj = error obj "unknown enum constant"


(* TODO, XXX: issue warning on unparsed fields or change behaviour depending on
 * "strict" config option ? *)<commit_msg>Fix: top-level enum value handling for Piqi tools and both OCaml and  Erlang mappings.<commit_after>(*
   Copyright 2009, 2010, 2011 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*)

(* Runtime support for piqi/Protocol Buffers wire format encoding
 *
 * Encoding rules follow this specification:
 *
 *   http://code.google.com/apis/protocolbuffers/docs/encoding.html
 *)


(*
 * Runtime support for parsers (decoders).
 *
 *)

exception Error of int * string


let string_of_loc pos =
  string_of_int pos


let strerr loc s = 
  string_of_loc loc ^ ": " ^ s


let buf_error loc s =
  (*
  failwith (strerr s loc)
  *)
  raise (Error (loc, s))


let error obj s =
  let loc = -1 in (* TODO, XXX: obj location db? *)
  buf_error loc s


module IBuf =
  struct
    type string_buf = 
      {
        s : string;
        start_pos : int;
        len :int;
        mutable pos : int; 
      }

    type stream_buf = char Stream.t

    type t =
      | Stream of stream_buf
      | String of string_buf


    let of_channel x =
      Stream (Stream.of_channel x)


    let of_string x start_pos =
      String
        { s = x; len = String.length x;
          start_pos = start_pos; pos = 0; 
        }


    let to_string buf =
      match buf with
        | Stream x ->
            let res = Buffer.create 20 in
            Stream.iter (Buffer.add_char res) x;
            Buffer.contents res
        | String x ->
            String.sub x.s x.pos (x.len - x.pos)


    let pos buf =
      match buf with
        | Stream x -> Stream.count x
        | String x -> x.pos + x.start_pos


    let error buf s =
      let loc = pos buf in
      buf_error loc s


    let next buf =
      try
        match buf with
          | Stream x -> Stream.next x
          | String x ->
              if x.pos >= x.len
              then
                raise Stream.Failure
              else
                let res = x.s.[x.pos] in
                x.pos <- x.pos + 1;
                res
      with Stream.Failure -> 
        error buf "unexpected end of block while reading integer"


    let skip_one buf =
      match buf with
        | Stream x -> Stream.junk x
        | String x -> x.pos <- x.pos + 1


    let is_empty buf =
      match buf with
        | Stream x ->
            Stream.peek x = None
        | String x ->
            x.pos = x.len


    let next_block buf length =
      try
        match buf with
          | Stream x ->
              let s = String.create length in
              let start_pos = Stream.count x in
              for i = 0 to length-1
              do
                s.[i] <- Stream.next x
              done;
              of_string s start_pos
          | String x ->
              if x.pos + length > x.len
              then
                (* XXX: adjusting position to provide proper EOB location *)
                (x.pos <- x.len; raise Stream.Failure)
              else
                (* NOTE: start_pos, pos and the string itself remain the same in
                 * the new buffer *)
                let res = String { x with len = x.pos + length } in
                (* skip the new buffer in the current buffer *)
                x.pos <- x.pos + length;
                res
      with Stream.Failure -> 
        error buf "unexpected end of block"


    let of_string x =
      of_string x 0
  end


type t = 
  | Varint of int
  | Varint64 of int64 (* used if int width is not enough *)
  | Int32 of int32
  | Int64 of int64
  | Block of IBuf.t
  | Top_block of IBuf.t (* top-level block *)


(* initializers for embedded records/variants (i.e. their contents start without
 * any leading headers/delimiters/separators) *)
let init_from_channel ch =
  Top_block (IBuf.of_channel ch)


let init_from_string s =
  Top_block (IBuf.of_string s)


let is_empty buf =
  IBuf.is_empty buf


let error_variant obj code =
  error obj ("unknown variant: " ^ string_of_int code)
let error_missing obj code =
  error obj  ("missing field " ^ string_of_int code)

let error_enum_const obj = error obj "unknown enum constant"


(* TODO, XXX: issue warning on unparsed fields or change behaviour depending on
 * "strict" config option ? *)
{'fraction_non_alphanumeric': 0.07568145376803848, 'fraction_numerical': 0.007910208444681988, 'mean_word_length': 1.7276967930029155, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '85138', 'n_tokens_mistral': 2829, 'n_tokens_neox': 2653, 'n_words': 1059}