<commit_before>(*
   Templates related to the data model designed by <NAME>.
   <NAME>, Cambridge University Computer Lab, March 2015
*)

open General
open State
open Crisp_syntax
open Naasty

type data_model_component =
  { name : string;
    identifiers : string list;
    type_scheme : naasty_type;
    function_scheme : naasty_function }

(*
  Based on
   https://lsds.doc.ic.ac.uk/gitlab/naas/naas-box-system/blob/master/src/applications/hadoop_data_model/HadoopDMData.h 

  static HadoopDMData *bytes_stream_to_channel(char *stream, char *channel,
    char *streamend, size_t *bytes_read, size_t *bytes_written);
  static void write_bytes_to_channel(HadoopDMData *,char *chan,
    size_t *no_bytes);
  static void bytes_channel_to_stream(char *stream, char *channel,
    size_t *bytes_read, size_t *bytes_written);
  size_t get_channel_len();
  size_t get_stream_len();
*)
let get_channel_len (datatype_name : string) (ty : Crisp_syntax.type_value) =
  let name = "get_channel_len" in
  { name = name;
    identifiers =
      [name;
       datatype_name ^ "::get_channel_len";
       "len";
       datatype_name;
       "sizeof";
      ];
    type_scheme = Fun_Type (-1, Size_Type None, []);
    function_scheme =
      let fun_name_idx = -2 in
      let arg_tys = [] in
      let ret_ty = Size_Type None in
      let body =
        [
          Declaration (Size_Type (Some (-3)));
          Assign (-3, Call_Function (-5, [Var (-4)]));
          (*FIXME fill in the rest of the body*)
          Return (Var (-3))
        ] |> Naasty_aux.concat
      in (fun_name_idx, arg_tys, ret_ty, body);
  }

let get_stream_len (datatype_name : string) (ty : Crisp_syntax.type_value) =
  { name = "get_stream_len";
    identifiers =
      ["get_stream_len";
       datatype_name ^ "::get_stream_len";
      ];<commit_msg>completed first-approximation behaviour for get_channel_len generator;<commit_after>(*
   Templates related to the data model designed by <NAME>.
   <NAME>, Cambridge University Computer Lab, March 2015
*)

open General
open State
open Crisp_syntax
open Naasty

type data_model_component =
  { name : string;
    identifiers : string list;
    type_scheme : naasty_type;
    function_scheme : naasty_function }

type type_analysis =
  Naasty.naasty_statement list * string list * identifier

(*
  Based on
   https://lsds.doc.ic.ac.uk/gitlab/naas/naas-box-system/blob/master/src/applications/hadoop_data_model/HadoopDMData.h 

  static HadoopDMData *bytes_stream_to_channel(char *stream, char *channel,
    char *streamend, size_t *bytes_read, size_t *bytes_written);
  static void write_bytes_to_channel(HadoopDMData *,char *chan,
    size_t *no_bytes);
  static void bytes_channel_to_stream(char *stream, char *channel,
    size_t *bytes_read, size_t *bytes_written);
  size_t get_channel_len();
  size_t get_stream_len();
*)
let get_channel_len (datatype_name : string) (ty : Crisp_syntax.type_value) =
  let name = "get_channel_len" in
  let identifiers =
    [name;
     datatype_name ^ "::get_channel_len";
     "len";
     datatype_name;
     "sizeof";
    ] in
  let rec analyse_type ((stmts, names, next_placeholder) as acc : type_analysis) ty : type_analysis =
    match ty with
    | RecordType (label_opt, tys, ty_ann) ->
      (*FIXME probably we should look at ty_ann*)
      snd(*FIXME code style*) (fold_map ([], acc) (fun acc ty -> ([], analyse_type acc ty)) tys)
    | String (label_opt, ty_ann) ->
      begin
        match List.filter (fun (k, v) -> k = "byte_size") ty_ann with
        | [] -> failwith "Strings need to be given an indication of size."
        | [(_, v)] ->
          begin
            match v with
            | Ann_Str _ -> failwith "TODO"
            | Ann_Int _ -> failwith "TODO"
            | Ann_Ident s ->
              let stmt =
                If1 (GEq (Var next_placeholder, Int_Value 0),
                     Increment (-3, Var next_placeholder))
              in (stmt :: stmts, s :: names, next_placeholder - 1)
          end
        | _ -> failwith "Too many sizes specified for a string."
      end
    | _ -> acc in
  let body_contents, more_idents, _ =
    analyse_type ([Skip], [], (List.length identifiers + 1) * (-1)) ty in
  { name = name;
    identifiers = identifiers @ more_idents;
    type_scheme = Fun_Type (-1, Size_Type None, []);
    function_scheme =
      let fun_name_idx = -2 in
      let arg_tys = [] in
      let ret_ty = Size_Type None in
      let body =
        [
          Declaration (Size_Type (Some (-3)));
          Assign (-3, Call_Function (-5, [Var (-4)]));
          Naasty_aux.concat body_contents;
          Return (Var (-3))
        ] |> Naasty_aux.concat
      in (fun_name_idx, arg_tys, ret_ty, body);
  }

let get_stream_len (datatype_name : string) (ty : Crisp_syntax.type_value) =
  { name = "get_stream_len";
    identifiers =
      ["get_stream_len";
       datatype_name ^ "::get_stream_len";
      ];
{'fraction_non_alphanumeric': 0.11068158697863682, 'fraction_numerical': 0.005696846388606307, 'mean_word_length': 2.8466353677621283, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1621198', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1611, 'n_words': 426}