<commit_before>type mode =
    NoEnrichment
  | Arguments
  | ArgumentsAndClosures
  | NamesResolved
  | VersionedResolved
  | PointsToResolved

let filter = ref false
let delta = ref false
let debug = ref false
let output = ref true

let maybe fmt arg =
  if !output then fmt Fmt.stdout arg

let time name fn x =
  Log.info (fun m -> m "Starting %s" name);
  let start = Sys.time () in
  let result = fn x in
  let stop = Sys.time () in
  Log.info (fun m -> m "Finished %s, took ~%f seconds" name (stop-.start));
  result

let calculate_filter_bound objects =
  let open Types in
  let seen = Hashtbl.create (BatDynArray.length objects) in
  let rec find_max max i =
    let max = Pervasives.max max i in
    let next = BatDynArray.get objects i in
      StringMap.fold (fun _ { value } (max: int) ->
                        if Hashtbl.mem seen value then
                          max
                        else begin
                          Hashtbl.add seen value ();
                          match try_get_object value with
                            | Some j -> find_max max j
                            | None -> max
                        end)
        next max
  in find_max 0 0 + 1

let rec list_delta init diff = function
  | [] -> []
  | value :: list ->
      diff init value :: list_delta value diff list

let delta_encode init diff list =
  let (ops, data) = List.split list in
    List.combine ops (list_delta init diff data)

let pp_enriched_trace init diff fmt fmtdiff pp list =
  let double_sep = Fmt.cut
  and with_sep = Fmt.always " with@ " in
  let list_fmt fmt =
    Fmt.vbox (Fmt.list ~sep:double_sep
                 (Fmt.vbox ~indent:2 (Fmt.pair ~sep:with_sep
                    TraceTypes.pp_clean_operation
                    fmt))) in
  if !delta then
    Fmt.using (delta_encode init diff) (list_fmt fmtdiff) pp list
  else
    list_fmt fmt pp list

type versions_resolved_delta = {
  last_arguments : int option;
  closures : Reference.reference StringMap.t ExtMap.diff IntMap.t;
  last_update : Reference.versioned_reference option;
  versions : int ExtMap.diff Reference.ReferenceMap.t;
  names : Reference.reference ExtMap.diff StringMap.t;
}
let filter_versions m =
  Reference.ReferenceMap.filter (fun key _ -> match key with
                                   | Reference.Field (Types.Object 0, _) -> true
                                   | Reference.Field (obj, _) -> Types.get_object_id obj >= !LocalFacts.filter_bound
                                   | _ -> true) m
let filter_points_to m =
  Reference.VersionedReferenceMap.filter (fun key _ -> match key with
                                   | (Reference.Field (Types.Object 0, _), _) -> true
                                   | (Reference.Field (obj, _), _) -> Types.get_object_id obj >= !LocalFacts.filter_bound
                                   | _ -> true) m

module FmtDiff(M: ExtMap.S) = struct
  let fmt pp_entry key pp map =
    if not (M.is_empty map) then
      Format.fprintf pp "@[<v 2>%s@ %a@]@ "
        key
        (M.pp ~pair_sep:(Fmt.always ": ")
           ~entry_sep:Fmt.cut (ExtMap.pp_diff pp_entry)) map
end
let fmt_versions pp versions =
  let module F = FmtDiff(Reference.ReferenceMap) in
    F.fmt Fmt.int "versions" pp (filter_versions versions)
let fmt_names =
  let module F = FmtDiff(StringMap) in
    F.fmt Reference.pp_reference "names"
let fmt_closures =
  let module F = FmtDiff(IntMap) in
    F.fmt (Fmt.braces (StringMap.pp Reference.pp_reference)) "closures"
let fmt_points_to pp pointsto =
  Reference.pp_versioned_reference_map
    (ExtMap.pp_diff Types.pp_jsval)
    pp (filter_points_to pointsto)
let fmt_option fmt key pp = function
  | Some x -> Format.fprintf pp "%s: %a@ " key fmt x
  | None -> ()
  

let pp_versions_resolved_delta pp 
      { last_arguments; closures; last_update; versions; names } =
  Format.pp_open_vbox pp 0;
  fmt_option Fmt.int "last_argument" pp last_arguments;
  fmt_option Reference.pp_versioned_reference "last_update" pp last_update;
  fmt_versions pp versions;
  fmt_names pp names;
  fmt_closures pp closures;
  Format.pp_close_box pp ()

let pp_enriched_trace_versions =
  let open LocalFacts in
  pp_enriched_trace 
    { last_arguments = None;
      closures = IntMap.empty;
      last_update = None;
      versions = Reference.ReferenceMap.empty;
      names = StringMap.empty }
    (fun { versions = old_versions; names = old_names; closures = old_closures }
           ({ versions; names; closures; last_update; last_arguments }) ->
       ({ last_update; last_arguments;
          closures = IntMap.delta (StringMap.equal (Reference.equal_reference)) old_closures closures;
          versions = Reference.ReferenceMap.delta (=) old_versions versions;
          names = StringMap.delta (Reference.equal_reference) old_names names }: versions_resolved_delta))
    pp_versions_resolved pp_versions_resolved_delta
    
type local_facts_delta = {
  last_arguments : int option;
  closures : Reference.reference StringMap.t ExtMap.diff IntMap.t;
  last_update : Reference.versioned_reference option;
  versions : int ExtMap.diff Reference.ReferenceMap.t;
  names : Reference.reference ExtMap.diff StringMap.t;
  points_to: Types.jsval ExtMap.diff Reference.VersionedReferenceMap.t
}
let pp_local_facts_delta pp 
      { last_arguments; closures; last_update; versions; names; points_to } =
  Format.pp_open_vbox pp 0;
  fmt_option Fmt.int "last_argument" pp last_arguments;
  fmt_option Reference.pp_versioned_reference "last_update" pp last_update;
  fmt_versions pp versions;
  fmt_names pp names;
  fmt_closures pp closures;
  fmt_points_to pp points_to;
  Format.pp_close_box pp ()

let pt_delta pt_old pt_new =
  let module M = Reference.VersionedReferenceMap in
    M.merge (fun _ vold vnew -> match vold, vnew with
               | Some xold, Some xnew ->
                   if xold = xnew then
                     None
                   else
                     Some (ExtMap.Change (xold, xnew))
               | Some xold, None -> Some (ExtMap.Remove xold)
               | None, Some xnew -> Some (ExtMap.Add xnew)
               | None, None -> None)
      pt_old pt_new

let pp_enriched_trace_points_to =
  let open LocalFacts in
  pp_enriched_trace 
    { last_arguments = None;
      closures = IntMap.empty;
      last_update = None;
      versions = Reference.ReferenceMap.empty;
      names = StringMap.empty;
      points_to = Reference.VersionedReferenceMap.empty ()
    }
    (fun { versions = old_versions; names = old_names; closures = old_closures;
           points_to = old_points_to }
           ({ versions; names; closures; last_update; last_arguments;<commit_msg>Port to new containers API.<commit_after>type mode =
    NoEnrichment
  | Arguments
  | ArgumentsAndClosures
  | NamesResolved
  | VersionedResolved
  | PointsToResolved

let filter = ref false
let delta = ref false
let debug = ref false
let output = ref true

let maybe fmt arg =
  if !output then fmt Fmt.stdout arg

let time name fn x =
  Log.info (fun m -> m "Starting %s" name);
  let start = Sys.time () in
  let result = fn x in
  let stop = Sys.time () in
  Log.info (fun m -> m "Finished %s, took ~%f seconds" name (stop-.start));
  result

let calculate_filter_bound objects =
  let open Types in
  let seen = Hashtbl.create (BatDynArray.length objects) in
  let rec find_max max i =
    let max = Pervasives.max max i in
    let next = BatDynArray.get objects i in
      StringMap.fold (fun _ { value } (max: int) ->
                        if Hashtbl.mem seen value then
                          max
                        else begin
                          Hashtbl.add seen value ();
                          match try_get_object value with
                            | Some j -> find_max max j
                            | None -> max
                        end)
        next max
  in find_max 0 0 + 1

let rec list_delta init diff = function
  | [] -> []
  | value :: list ->
      diff init value :: list_delta value diff list

let delta_encode init diff list =
  let (ops, data) = List.split list in
    List.combine ops (list_delta init diff data)

let pp_enriched_trace init diff fmt fmtdiff pp list =
  let double_sep = Fmt.cut
  and with_sep = Fmt.always " with@ " in
  let list_fmt fmt =
    Fmt.vbox (Fmt.list ~sep:double_sep
                 (Fmt.vbox ~indent:2 (Fmt.pair ~sep:with_sep
                    TraceTypes.pp_clean_operation
                    fmt))) in
  if !delta then
    Fmt.using (delta_encode init diff) (list_fmt fmtdiff) pp list
  else
    list_fmt fmt pp list

type versions_resolved_delta = {
  last_arguments : int option;
  closures : Reference.reference StringMap.t ExtMap.diff IntMap.t;
  last_update : Reference.versioned_reference option;
  versions : int ExtMap.diff Reference.ReferenceMap.t;
  names : Reference.reference ExtMap.diff StringMap.t;
}
let filter_versions m =
  Reference.ReferenceMap.filter (fun key _ -> match key with
                                   | Reference.Field (Types.Object 0, _) -> true
                                   | Reference.Field (obj, _) -> Types.get_object_id obj >= !LocalFacts.filter_bound
                                   | _ -> true) m
let filter_points_to m =
  Reference.VersionedReferenceMap.filter (fun key _ -> match key with
                                   | (Reference.Field (Types.Object 0, _), _) -> true
                                   | (Reference.Field (obj, _), _) -> Types.get_object_id obj >= !LocalFacts.filter_bound
                                   | _ -> true) m

module FmtDiff(M: ExtMap.S) = struct
  let fmt pp_entry key pp map =
    if not (M.is_empty map) then
      Format.fprintf pp "@[<v 2>%s@ %a@]@ "
        key
        (M.pp ~pair_sep:(Fmt.always ": ")
           ~entry_sep:Fmt.cut (ExtMap.pp_diff pp_entry)) map
end
let fmt_versions pp versions =
  let module F = FmtDiff(Reference.ReferenceMap) in
    F.fmt Fmt.int "versions" pp (filter_versions versions)
let fmt_names =
  let module F = FmtDiff(StringMap) in
    F.fmt Reference.pp_reference "names"
let fmt_closures =
  let module F = FmtDiff(IntMap) in
    F.fmt (Fmt.braces (StringMap.pp Reference.pp_reference)) "closures"
let fmt_points_to pp pointsto =
  Reference.pp_versioned_reference_map
    (ExtMap.pp_diff Types.pp_jsval)
    pp (filter_points_to pointsto)
let fmt_option fmt key pp = function
  | Some x -> Format.fprintf pp "%s: %a@ " key fmt x
  | None -> ()
  

let pp_versions_resolved_delta pp 
      { last_arguments; closures; last_update; versions; names } =
  Format.pp_open_vbox pp 0;
  fmt_option Fmt.int "last_argument" pp last_arguments;
  fmt_option Reference.pp_versioned_reference "last_update" pp last_update;
  fmt_versions pp versions;
  fmt_names pp names;
  fmt_closures pp closures;
  Format.pp_close_box pp ()

let pp_enriched_trace_versions =
  let open LocalFacts in
  pp_enriched_trace 
    { last_arguments = None;
      closures = IntMap.empty;
      last_update = None;
      versions = Reference.ReferenceMap.empty;
      names = StringMap.empty }
    (fun { versions = old_versions; names = old_names; closures = old_closures }
           ({ versions; names; closures; last_update; last_arguments }) ->
       ({ last_update; last_arguments;
          closures = IntMap.delta (StringMap.equal (Reference.equal_reference)) old_closures closures;
          versions = Reference.ReferenceMap.delta (=) old_versions versions;
          names = StringMap.delta (Reference.equal_reference) old_names names }: versions_resolved_delta))
    pp_versions_resolved pp_versions_resolved_delta
    
type local_facts_delta = {
  last_arguments : int option;
  closures : Reference.reference StringMap.t ExtMap.diff IntMap.t;
  last_update : Reference.versioned_reference option;
  versions : int ExtMap.diff Reference.ReferenceMap.t;
  names : Reference.reference ExtMap.diff StringMap.t;
  points_to: Types.jsval ExtMap.diff Reference.VersionedReferenceMap.t
}
let pp_local_facts_delta pp 
      { last_arguments; closures; last_update; versions; names; points_to } =
  Format.pp_open_vbox pp 0;
  fmt_option Fmt.int "last_argument" pp last_arguments;
  fmt_option Reference.pp_versioned_reference "last_update" pp last_update;
  fmt_versions pp versions;
  fmt_names pp names;
  fmt_closures pp closures;
  fmt_points_to pp points_to;
  Format.pp_close_box pp ()

let pt_delta pt_old pt_new =
  let module M = Reference.VersionedReferenceMap in
    M.merge (fun _ -> function
               | `Both (xold, xnew) ->
                   if xold = xnew then
                     None
                   else
                     Some (ExtMap.Change (xold, xnew))
               | `Left xold -> Some (ExtMap.Remove xold)
               | `Right xnew -> Some (ExtMap.Add xnew))
      pt_old pt_new

let pp_enriched_trace_points_to =
  let open LocalFacts in
  pp_enriched_trace 
    { last_arguments = None;
      closures = IntMap.empty;
      last_update = None;
      versions = Reference.ReferenceMap.empty;
      names = StringMap.empty;
      points_to = Reference.VersionedReferenceMap.empty ()
    }
    (fun { versions = old_versions; names = old_names; closures = old_closures;
           points_to = old_points_to }
           ({ versions; names; closures; last_update; last_arguments;
{'fraction_non_alphanumeric': 0.0815666817019997, 'fraction_numerical': 0.0013531799729364006, 'mean_word_length': 2.4508430609597927, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6497323', 'n_tokens_mistral': 4113, 'n_tokens_neox': 4014, 'n_words': 1168}