<commit_before>
let debug = false

open Code

(****)

let add_var vars x = vars := VarSet.add x !vars

type def = Phi of VarSet.t | Expr of Code.expr | Param

let undefined = Phi VarSet.empty

let is_undefined d = match d with Phi s -> VarSet.is_empty s | _ -> false

let add_expr_def defs x e =
  let idx = Var.idx x in
  assert (is_undefined defs.(idx));
  defs.(idx) <- Expr e

let add_assign_def vars defs x y =
  add_var vars x;
  let idx = Var.idx x in
  match defs.(idx) with
    Expr _ | Param ->
      assert false
  | Phi s  ->
      defs.(idx) <- Phi (VarSet.add y s)

let add_param_def vars defs x =
  add_var vars x;
  let idx = Var.idx x in
  assert (is_undefined defs.(idx) || defs.(idx) = Param);
  defs.(idx) <- Param

(* x depends on y *)
let add_dep deps x y =
  let idx = Var.idx y in
  deps.(idx) <- VarSet.add x deps.(idx)

let rec arg_deps vars deps defs params args =
  match params, args with
    x :: params, y :: args ->
      add_dep deps x y;
      add_assign_def vars defs x y;
      arg_deps vars deps defs params args
  | _ ->
      ()

let cont_deps blocks vars deps defs (pc, args) =
  let block = AddrMap.find pc blocks in
  arg_deps vars deps defs block.params args

let expr_deps blocks vars deps defs x e =
  match e with
    Const _ | Constant _ | Apply _ | Direct_apply _ | Prim _ ->
      ()
  | Closure (l, cont) ->
      List.iter (fun x -> add_param_def vars defs x) l;
      cont_deps blocks vars deps defs cont
  | Block (_, a) ->
      Array.iter (fun y -> add_dep deps x y) a
  | Field (y, _) ->
      add_dep deps x y
  | Variable y ->
      assert false

let program_deps (_, blocks, _) =
  let nv = Var.count () in
  let vars = ref VarSet.empty in
  let deps = Array.make nv VarSet.empty in
  let defs = Array.make nv undefined in
  AddrMap.iter
    (fun pc block ->
       List.iter
         (fun i ->
            match i with
              Let (x, e) ->
                add_var vars x;
                add_expr_def defs x e;
                expr_deps blocks vars deps defs x e
            | Set_field _ | Array_set _ | Offset_ref _ ->
                ())
         block.body;
       Util.opt_iter
         (fun (x, cont) ->
            add_param_def vars defs x;
            cont_deps blocks vars deps defs cont)
         block.handler;
       match block.branch with
         Return _ | Raise _ | Stop ->
           ()
       | Branch cont ->
           cont_deps blocks vars deps defs cont
       | Cond (_, _, cont1, cont2) ->
           cont_deps blocks vars deps defs cont1;
           cont_deps blocks vars deps defs cont2
       | Switch (_, a1, a2) ->
           Array.iter (fun cont -> cont_deps blocks vars deps defs cont) a1;
           Array.iter (fun cont -> cont_deps blocks vars deps defs cont) a2
       | Pushtrap (cont, _, _, _) ->
           cont_deps blocks vars deps defs cont
       | Poptrap cont ->
           cont_deps blocks vars deps defs cont)
    blocks;
  (vars, deps, defs)

let propagate1 deps defs st x =
  match defs.(Var.idx x) with
    Param ->
      VarSet.empty
  | Phi s ->
      VarSet.fold (fun y s -> VarSet.union (VarMap.find y st) s) s VarSet.empty
  | Expr e ->
      match e with
        Const _ | Constant _  | Apply _ | Direct_apply _ | Prim _ ->
          VarSet.singleton x
      | Closure _ | Block _ ->
          VarSet.singleton x
      | Field (y, n) ->
          let s = VarMap.find y st in
          VarSet.fold
            (fun z s ->
               match defs.(Var.idx z) with
                 Phi _ | Param ->
                   assert false
               | Expr (Block (_, a)) ->
(*Format.eprintf "%d/%d@." n (Array.length a);*)
                   if n >= Array.length a then s else begin
                     let t = a.(n) in
(*Format.eprintf "%a --> %a@." Var.print t Var.print x;*)
                     add_dep deps x t;
                     VarSet.union (VarMap.find t st) s
                   end<commit_msg>-<commit_after>
let debug = false

open Code

(****)

let add_var vars x = vars := VarSet.add x !vars

type def = Phi of VarSet.t | Expr of Code.expr | Param

let undefined = Phi VarSet.empty

let is_undefined d = match d with Phi s -> VarSet.is_empty s | _ -> false

let add_expr_def defs x e =
  let idx = Var.idx x in
  assert (is_undefined defs.(idx));
  defs.(idx) <- Expr e

let add_assign_def vars defs x y =
  add_var vars x;
  let idx = Var.idx x in
  match defs.(idx) with
    Expr _ | Param ->
      assert false
  | Phi s  ->
      defs.(idx) <- Phi (VarSet.add y s)

let add_param_def vars defs x =
  add_var vars x;
  let idx = Var.idx x in
  assert (is_undefined defs.(idx) || defs.(idx) = Param);
  defs.(idx) <- Param

(* x depends on y *)
let add_dep deps x y =
  let idx = Var.idx y in
  deps.(idx) <- VarSet.add x deps.(idx)

let rec arg_deps vars deps defs params args =
  match params, args with
    x :: params, y :: args ->
      add_dep deps x y;
      add_assign_def vars defs x y;
      arg_deps vars deps defs params args
  | _ ->
      ()

let cont_deps blocks vars deps defs (pc, args) =
  let block = AddrMap.find pc blocks in
  arg_deps vars deps defs block.params args

let expr_deps blocks vars deps defs x e =
  match e with
    Const _ | Constant _ | Apply _ | Direct_apply _ | Prim _ ->
      ()
  | Closure (l, cont) ->
      List.iter (fun x -> add_param_def vars defs x) l;
      cont_deps blocks vars deps defs cont
  | Block (_, a) ->
      Array.iter (fun y -> add_dep deps x y) a
  | Field (y, _) ->
      add_dep deps x y
  | Variable y ->
      assert false

let program_deps (_, blocks, _) =
  let nv = Var.count () in
  let vars = ref VarSet.empty in
  let deps = Array.make nv VarSet.empty in
  let defs = Array.make nv undefined in
  AddrMap.iter
    (fun pc block ->
       List.iter
         (fun i ->
            match i with
              Let (x, e) ->
                add_var vars x;
                add_expr_def defs x e;
                expr_deps blocks vars deps defs x e
            | Set_field _ | Array_set _ | Offset_ref _ ->
                ())
         block.body;
       Util.opt_iter
         (fun (x, cont) ->
            add_param_def vars defs x;
            cont_deps blocks vars deps defs cont)
         block.handler;
       match block.branch with
         Return _ | Raise _ | Stop ->
           ()
       | Branch cont ->
           cont_deps blocks vars deps defs cont
       | Cond (_, _, cont1, cont2) ->
           cont_deps blocks vars deps defs cont1;
           cont_deps blocks vars deps defs cont2
       | Switch (_, a1, a2) ->
           Array.iter (fun cont -> cont_deps blocks vars deps defs cont) a1;
           Array.iter (fun cont -> cont_deps blocks vars deps defs cont) a2
       | Pushtrap (cont, _, _, _) ->
           cont_deps blocks vars deps defs cont
       | Poptrap cont ->
           cont_deps blocks vars deps defs cont)
    blocks;
  (vars, deps, defs)

let propagate1 deps defs st x =
  match defs.(Var.idx x) with
    Param ->
      VarSet.empty
  | Phi s ->
      VarSet.fold (fun y s -> VarSet.union (VarMap.find y st) s) s VarSet.empty
  | Expr e ->
      match e with
        Const _ | Constant _  | Apply _ | Direct_apply _ | Prim _ ->
          VarSet.singleton x
      | Closure _ | Block _ ->
          VarSet.singleton x
      | Field (y, n) ->
          let s = VarMap.find y st in
          VarSet.fold
            (fun z s ->
               match defs.(Var.idx z) with
                 Phi _ | Param ->
                   assert false
               | Expr (Block (_, a)) ->
(*Format.eprintf "%d/%d@." n (Array.length a);*)
                   if n >= Array.length a then s else begin
                     let t = a.(n) in
(*Format.eprintf "%a --> %a@." Var.print t Var.print x;*)
                     add_dep deps x t;
                     let s' = VarMap.find t st in
                     if VarSet.is_empty s' then
                       VarSet.add t s'
                     else
                       VarSet.union s' s
                   end
{'fraction_non_alphanumeric': 0.09560755850331623, 'fraction_numerical': 0.0022525341008634713, 'mean_word_length': 1.6384945526576429, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2900191', 'n_tokens_mistral': 2706, 'n_tokens_neox': 2660, 'n_words': 941}