<commit_before>(** A type, with free type variables for polymorphic arguments. *)
open Types
open SmartPrint
open Monad.Notations

type t =
  | Variable of Name.t
  | Arrow of t * t
  | Sum of (string * t) list
  | Tuple of t list
  | Apply of MixedPath.t * t list
  | Package of bool * PathName.t * arity_or_typ Tree.t
  | ForallModule of Name.t * t * t
  | ForallTyps of Name.t list * t
  | FunTyps of Name.t list * t
  | Error of string

and arity_or_typ =
  | Arity of int
  | Typ of t

let type_exprs_of_row_field (row_field : Types.row_field)
  : Types.type_expr list =
  match row_field with
  | Rpresent None -> []
  | Rpresent (Some typ) -> [typ]
  | Reither (_, typs, _, _) -> typs
  | Rabsent -> []

let filter_typ_params_in_valid_set
  (typ_params : Name.t list) (valid_set : Name.Set.t) : bool list =
  typ_params |> List.map (fun ty -> Name.Set.mem ty valid_set)


let rec non_phantom_typs (path : Path.t) (typs : Types.type_expr list)
  : Types.type_expr list Monad.t =
  get_env >>= fun env ->
  begin match Env.find_type path env with
  | typ_declaration ->
    AdtParameters.of_ocaml typ_declaration.type_params >>= fun typ_params ->
    Attribute.of_attributes typ_declaration.type_attributes >>= fun typ_attributes ->
    let is_phantom = Attribute.has_phantom typ_attributes in
    if is_phantom then
      return (Some (typ_params |> List.map (fun _ -> false)))
    else
      begin match typ_declaration.type_kind with
      | Type_abstract ->
        begin match typ_declaration.type_manifest with
        | None ->
          return (Some (typ_params |> List.map (function
            | AdtParameters.AdtVariable.Parameter _ -> true
            | _ ->
              if Path.name path = "array" then
                true
              else
                false
          )))
        (* Specific case for inductives defined with polymorphic variants. *)
        | Some { desc = Tvariant _; _ } ->
          return (Some (typ_params |> List.map (function
            | AdtParameters.AdtVariable.Parameter _ -> true
            | _ -> false
          )))
        | Some typ ->
          non_phantom_vars_of_typ typ >>= fun non_phantom_typ_vars ->
          let typ_params = AdtParameters.get_parameters typ_params in
          return (Some (
            filter_typ_params_in_valid_set typ_params non_phantom_typ_vars
          ))
        end
      | Type_record (labels, _) ->
        let typs = List.map (fun label -> label.ld_type) labels in
        non_phantom_vars_of_typs typs >>= fun non_phantom_typ_vars ->
        let typ_params = AdtParameters.get_parameters typ_params in
        return (Some (
          filter_typ_params_in_valid_set typ_params non_phantom_typ_vars
        ))
      | Type_variant constructors ->
        let* constructors_return_typ_params =
          constructors |> Monad.List.map (fun constructor ->
              AdtParameters.get_return_typ_params typ_params constructor.cd_res
            ) in
        let gadt_shape = AdtParameters.gadt_shape typ_params constructors_return_typ_params in

        return (Some (gadt_shape |> List.map (fun shape ->
            match shape with
            | None -> Attribute.has_force_gadt typ_attributes
            | Some _ -> true
          )))
      | Type_open -> return None
      end
    | exception Not_found -> return None
    end >>= fun non_phantom_typs_shape ->
  let typs =
    match non_phantom_typs_shape with
    | None -> typs
    | Some non_phantom_typs_shape ->
      List.rev @@ List.fold_left2
        (fun typs is_non_phantom typ ->
          if is_non_phantom then
            typ :: typs
          else
            typs<commit_msg>Fix filter_typ_params_in_valid_set to consider indexes<commit_after>(** A type, with free type variables for polymorphic arguments. *)
open Types
open SmartPrint
open Monad.Notations

type t =
  | Variable of Name.t
  | Arrow of t * t
  | Sum of (string * t) list
  | Tuple of t list
  | Apply of MixedPath.t * t list
  | Package of bool * PathName.t * arity_or_typ Tree.t
  | ForallModule of Name.t * t * t
  | ForallTyps of Name.t list * t
  | FunTyps of Name.t list * t
  | Error of string

and arity_or_typ =
  | Arity of int
  | Typ of t

let type_exprs_of_row_field (row_field : Types.row_field)
  : Types.type_expr list =
  match row_field with
  | Rpresent None -> []
  | Rpresent (Some typ) -> [typ]
  | Reither (_, typs, _, _) -> typs
  | Rabsent -> []

let filter_typ_params_in_valid_set
  (typ_params : AdtParameters.AdtVariable.t list) (valid_set : Name.Set.t) : bool list =
  typ_params |> List.map (function
      | AdtParameters.AdtVariable.Parameter name -> Name.Set.mem name valid_set
      | _ -> false )


let rec non_phantom_typs (path : Path.t) (typs : Types.type_expr list)
  : Types.type_expr list Monad.t =
  get_env >>= fun env ->
  begin match Env.find_type path env with
  | typ_declaration ->
    AdtParameters.of_ocaml typ_declaration.type_params >>= fun typ_params ->
    Attribute.of_attributes typ_declaration.type_attributes >>= fun typ_attributes ->
    let is_phantom = Attribute.has_phantom typ_attributes in
    if is_phantom then
      return (Some (typ_params |> List.map (fun _ -> false)))
    else
      begin match typ_declaration.type_kind with
      | Type_abstract ->
        begin match typ_declaration.type_manifest with
        | None ->
          return (Some (typ_params |> List.map (function
            | AdtParameters.AdtVariable.Parameter _ -> true
            | _ ->
              if Path.name path = "array" then
                true
              else
                false
          )))
        (* Specific case for inductives defined with polymorphic variants. *)
        | Some { desc = Tvariant _; _ } ->
          return (Some (typ_params |> List.map (function
            | AdtParameters.AdtVariable.Parameter _ -> true
            | _ -> false
          )))
        | Some typ ->
          non_phantom_vars_of_typ typ >>= fun non_phantom_typ_vars ->
          return (Some (
            filter_typ_params_in_valid_set typ_params non_phantom_typ_vars
          ))
        end
      | Type_record (labels, _) ->
        let typs = List.map (fun label -> label.ld_type) labels in
        non_phantom_vars_of_typs typs >>= fun non_phantom_typ_vars ->
        return (Some (
          filter_typ_params_in_valid_set typ_params non_phantom_typ_vars
        ))
      | Type_variant constructors ->
        let* constructors_return_typ_params =
          constructors |> Monad.List.map (fun constructor ->
              AdtParameters.get_return_typ_params typ_params constructor.cd_res
            ) in
        let gadt_shape = AdtParameters.gadt_shape typ_params constructors_return_typ_params in
        return (Some (gadt_shape |> List.map (fun shape ->
            match shape with
            | None -> Attribute.has_force_gadt typ_attributes
            | Some _ -> true
          )))
      | Type_open -> return None
      end
    | exception Not_found -> return None
    end >>= fun non_phantom_typs_shape ->
  let typs =
    match non_phantom_typs_shape with
    | None -> typs
    | Some non_phantom_typs_shape ->
      List.rev @@ List.fold_left2
        (fun typs is_non_phantom typ ->
          if is_non_phantom then
            typ :: typs
          else
            typs
{'fraction_non_alphanumeric': 0.08299039780521263, 'fraction_numerical': 0.00027434842249657066, 'mean_word_length': 2.256364448414471, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1113005', 'n_tokens_mistral': 2293, 'n_tokens_neox': 2234, 'n_words': 595}