<commit_before>#include "topdeps.h"

#include "types.h"
#include "parser.h"

struct topdeps {
  struct vectyp list;
  struct fintypset set;

  struct vectyploc tentatives;
};

static void record_final(struct module *mod, struct typ *t) {
  struct top_state *st = mod->state->top_state;
  struct fun_state *fun_st = mod->state->fun_state;

  struct node *topmost = st->top;
  while (!node_is_at_top(topmost)) {
    topmost = parent(topmost);
  }

  if (typ_is_pseudo_builtin(t)
      // when a try_remove_unnecessary_ssa_defname() is used on a global let:
      || topmost->which == NOOP
      || (st->top->typ != NULL && typ_equal(st->top->typ, t))
      || (topmost->typ != NULL && typ_equal(topmost->typ, t))) {
    return;
  }

  struct node *top = st->top;
  struct toplevel *toplevel = node_toplevel(top);

  static const uint32_t TO_KEEP = TOP_IS_EXPORT | TOP_IS_INLINE
    | TOP_IS_FUNCTOR | TOP_IS_PREVENT_DYN;
  static const uint32_t WEAK = TOP_IS_FUNCTOR;

  uint32_t mask = toplevel->flags & TO_KEEP;

  const bool within_exportable_field = st->exportable != NULL
    && st->exportable->which == DEFFIELD;

  if (within_exportable_field) {
    if (!name_is_export(mod, st->exportable)) {
      mask &= ~TOP_IS_EXPORT;
    }
  }
  if (fun_st != NULL) {
    if (fun_st->in_block) {
      if (!(mask & TOP_IS_INLINE)) {
        mask &= ~(TOP_IS_EXPORT | TOP_IS_INLINE);
      }
    }
  }

  if (!typ_is_reference(t)) {
    switch (top->which) {
    case DEFTYPE:
      if (within_exportable_field) {
        mask |= TOP__TOPDEP_INLINE_STRUCT;
      }
      break;
    case LET:
      if (subs_first_const(top)->which == DEFNAME) {
        mask |= TOP__TOPDEP_INLINE_STRUCT;
      }
      break;
    default:
      break;
    }
  }

  // This may prove to be too restrictive, but we want to make sure that
  // builtins appears first in generated code.
  const bool is_dependency_on_builtins = top->typ != NULL
    && strncmp(node_module_owner_const(top)->filename, "lib/n/builtins/",
               sizeof("lib/n/builtins/")-1) == 0;
  if (!is_dependency_on_builtins
      && typ_generic_arity(t) > 0 && !typ_is_generic_functor(t)) {
    // Inherit the inline struct dependencies from fields of a generic.
    struct tit *tit = typ_definition_members(t, DEFFIELD, 0);
    while (tit_next(tit)) {
      struct typ *fieldt = tit_typ(tit);
      if (fieldt == NULL) {
        continue;
      }
      struct module *owner = typ_module_owner(fieldt);
      if (owner != mod) {
        struct node *import = module_find_import(mod, owner);
        if (import != NULL) {
          node_toplevel(import)->flags |= TOP_IS_INLINE;
        }
      }
    }
  }

  // Attribute the dependency to the parent type (if any), unless 'top' is a
  // generic function.
  if (!node_is_at_top(top)
      && (!(NM(top->which) & (NM(DEFFUN) | NM(DEFMETHOD)))
          || typ_generic_arity(top->typ) == 0
          || typ_is_generic_functor(top->typ))) {
    const uint32_t member_mask = mask;

    top = parent(top);
    toplevel = node_toplevel(top);
    mask |= toplevel->flags & TO_KEEP;

    if (!(member_mask & TOP_IS_EXPORT) || !(toplevel->flags & TOP_IS_EXPORT)) {
      mask &= ~TOP_IS_EXPORT;
    }
    if (!(member_mask & TOP_IS_INLINE) || !(toplevel->flags & TOP_IS_INLINE)) {
      mask &= ~TOP_IS_INLINE;
    }
  }

  if (top->typ != NULL && typ_equal(top->typ, t)) {
    return;
  }

  if (toplevel->topdeps == NULL) {
    toplevel->topdeps = calloc(1, sizeof(*toplevel->topdeps));
    fintypset_fullinit(&toplevel->topdeps->set);
  }

  uint32_t *value = fintypset_get(&toplevel->topdeps->set, t);
  if (value == NULL) {
    fintypset_set(&toplevel->topdeps->set, t, mask);
    vectyp_push(&toplevel->topdeps->list, t);
  } else {
    // A weak flag can only be set by the initial add. Otherwise, it can
    // only be unset.
    if ((*value & WEAK) && (mask & WEAK)) {
      *value |= mask;
    } else {
      *value = (*value & ~WEAK) | (mask & ~WEAK);
    }
  }
}

static void record_tentative(struct module *mod, struct typ **loc) {
  struct top_state *st = mod->state->top_state;
  struct node *top = st->top;
  if (typ_is_pseudo_builtin(*loc)
      || (!typ_is_tentative(*loc)
          && !typ_is_ungenarg(*loc)
          && typ_definition_nooverlay(*loc) == top)) {
    return;
  }
  struct toplevel *toplevel = node_toplevel(top);
  if (toplevel->topdeps == NULL) {
    toplevel->topdeps = calloc(1, sizeof(*toplevel->topdeps));
    fintypset_fullinit(&toplevel->topdeps->set);
  }

  vectyploc_push(&toplevel->topdeps->tentatives, loc);
}

void topdeps_record(struct module *mod, struct typ *t) {
  struct top_state *st = mod->state->top_state;
  if (st == NULL
      || typ_definition_which(t) == MODULE) {
    return;
  }

  if (typ_is_tentative(t) || !typ_hash_ready(t)) {
    record_tentative(mod, typ_permanent_loc(t));
  } else {
    record_final(mod, t);
  }
}

error topdeps_foreach(struct module *mod, struct node *node,
                      topdeps_each each, void *user) {
  struct toplevel *toplevel = node_toplevel(node);
  if (toplevel->topdeps == NULL) {
    return 0;
  }

  // Need to work when entries are being added to the topdeps while we are
  // iterating over it, so we have to get the current count every time.

  struct vectyploc *tentatives = &toplevel->topdeps->tentatives;

  struct top_state *st = mod->state->top_state;
  // 'st' may be NULL when topdeps_foreach() is called from a non-passing
  // context, e.g. cprinter.

  for (ssize_t n = 0; n < vectyploc_count(tentatives); ++n) {
    struct typ *t = **vectyploc_get(tentatives, n);
    if (st != NULL) {
      if (!typ_is_tentative(t) && typ_hash_ready(t)) {
        record_final(mod, t);
        n += vectyploc_remove_replace_with_last(tentatives, n);
        continue;
      }
    }

    error e = each(mod, node, t, 0, user);
    EXCEPT(e);
  }

  struct vectyp *list = &toplevel->topdeps->list;<commit_msg>bootstrap:topdeps: if tentative, we must take a snapshot of the context<commit_after>#include "topdeps.h"

#include "types.h"
#include "parser.h"

// If the typ is tentative, we cannot record it yet, as its hash will
// change. We need to keep a snapshot of the context when the topdep was
// first recorded.
struct snapshot {
  struct typ **loc;

  struct node *top;
  struct node *exportable;
  bool in_fun_in_block;
};

VECTOR(vecsnapshot, struct snapshot, 4);
IMPLEMENT_VECTOR(unused__ static, vecsnapshot, struct snapshot);

// Loose elements ordering.
static inline use_result__ ssize_t vecsnapshot_remove_replace_with_last(struct vecsnapshot *v, ssize_t n) {
  const size_t count = vecsnapshot_count(v);
  if (count == 1) {
    vecsnapshot_destroy(v);
    return 0;
  }

  struct snapshot last = vecsnapshot_pop(v);
  if (n + 1 == count) {
    (void) last;
    return 0;
  } else {
    struct snapshot *snap = vecsnapshot_get(v, n);
    *snap = last;
    return -1;
  }
}

struct topdeps {
  struct vectyp list;
  struct fintypset set;

  struct vecsnapshot tentatives;
};

static void record_final(struct module *mod, struct snapshot *snap) {
  struct typ *t = *snap->loc;
  struct node *top = snap->top;

  struct node *topmost = top;
  while (!node_is_at_top(topmost)) {
    topmost = parent(topmost);
  }

  if (typ_is_pseudo_builtin(t)
      // when a try_remove_unnecessary_ssa_defname() is used on a global let:
      || topmost->which == NOOP
      || (top->typ != NULL && typ_equal(top->typ, t))
      || (topmost->typ != NULL && typ_equal(topmost->typ, t))) {
    return;
  }

  struct toplevel *toplevel = node_toplevel(top);

  static const uint32_t TO_KEEP = TOP_IS_EXPORT | TOP_IS_INLINE
    | TOP_IS_FUNCTOR | TOP_IS_PREVENT_DYN;
  static const uint32_t WEAK = TOP_IS_FUNCTOR;

  uint32_t mask = toplevel->flags & TO_KEEP;

  const bool within_exportable_field = snap->exportable != NULL
    && snap->exportable->which == DEFFIELD;
  if (within_exportable_field) {
    if (!name_is_export(mod, snap->exportable)) {
      mask &= ~TOP_IS_EXPORT;
    }
  }

  if (snap->in_fun_in_block) {
    if (!(mask & TOP_IS_INLINE)) {
      mask &= ~(TOP_IS_EXPORT | TOP_IS_INLINE);
    }
  }

  if (!typ_is_reference(t)) {
    switch (top->which) {
    case DEFTYPE:
      if (within_exportable_field) {
        mask |= TOP__TOPDEP_INLINE_STRUCT;
      }
      break;
    case LET:
      if (subs_first_const(top)->which == DEFNAME) {
        mask |= TOP__TOPDEP_INLINE_STRUCT;
      }
      break;
    default:
      break;
    }
  }

  // This may prove to be too restrictive, but we want to make sure that
  // builtins appears first in generated code.
  const bool is_dependency_on_builtins = top->typ != NULL
    && strncmp(node_module_owner_const(top)->filename, "lib/n/builtins/",
               sizeof("lib/n/builtins/")-1) == 0;
  if (!is_dependency_on_builtins
      && typ_generic_arity(t) > 0 && !typ_is_generic_functor(t)) {
    // Inherit the inline struct dependencies from fields of a generic.
    struct tit *tit = typ_definition_members(t, DEFFIELD, 0);
    while (tit_next(tit)) {
      struct typ *fieldt = tit_typ(tit);
      if (fieldt == NULL) {
        continue;
      }
      struct module *owner = typ_module_owner(fieldt);
      if (owner != mod) {
        struct node *import = module_find_import(mod, owner);
        if (import != NULL) {
          node_toplevel(import)->flags |= TOP_IS_INLINE;
        }
      }
    }
  }

  // Attribute the dependency to the parent type (if any), unless 'top' is a
  // generic function.
  if (!node_is_at_top(top)
      && (!(NM(top->which) & (NM(DEFFUN) | NM(DEFMETHOD)))
          || typ_generic_arity(top->typ) == 0
          || typ_is_generic_functor(top->typ))) {
    const uint32_t member_mask = mask;

    top = parent(top);
    toplevel = node_toplevel(top);
    mask |= toplevel->flags & TO_KEEP;

    if (!(member_mask & TOP_IS_EXPORT) || !(toplevel->flags & TOP_IS_EXPORT)) {
      mask &= ~TOP_IS_EXPORT;
    }
    if (!(member_mask & TOP_IS_INLINE) || !(toplevel->flags & TOP_IS_INLINE)) {
      mask &= ~TOP_IS_INLINE;
    }
  }

  if (top->typ != NULL && typ_equal(top->typ, t)) {
    return;
  }

  if (toplevel->topdeps == NULL) {
    toplevel->topdeps = calloc(1, sizeof(*toplevel->topdeps));
    fintypset_fullinit(&toplevel->topdeps->set);
  }

  uint32_t *value = fintypset_get(&toplevel->topdeps->set, t);
  if (value == NULL) {
    fintypset_set(&toplevel->topdeps->set, t, mask);
    vectyp_push(&toplevel->topdeps->list, t);
  } else {
    // A weak flag can only be set by the initial add. Otherwise, it can
    // only be unset.
    if ((*value & WEAK) && (mask & WEAK)) {
      *value |= mask;
    } else {
      *value = (*value & ~WEAK) | (mask & ~WEAK);
    }
  }
}

static void record_tentative(struct module *mod, struct snapshot *snap) {
  struct typ **loc = snap->loc;
  if (typ_is_pseudo_builtin(*loc)
      || (!typ_is_tentative(*loc)
          && !typ_is_ungenarg(*loc)
          && typ_definition_nooverlay(*loc) == snap->top)) {
    return;
  }

  struct toplevel *toplevel = node_toplevel(snap->top);
  if (toplevel->topdeps == NULL) {
    toplevel->topdeps = calloc(1, sizeof(*toplevel->topdeps));
    fintypset_fullinit(&toplevel->topdeps->set);
  }

  vecsnapshot_push(&toplevel->topdeps->tentatives, *snap);
}

void topdeps_record(struct module *mod, struct typ *t) {
  struct top_state *st = mod->state->top_state;
  if (st == NULL
      || typ_definition_which(t) == MODULE) {
    return;
  }

  struct fun_state *fun_st = mod->state->fun_state;
  struct snapshot snap = {
    .loc = typ_permanent_loc(t),
    .top = st->top,
    .exportable = st->exportable,
    .in_fun_in_block = fun_st != NULL && fun_st->in_block,
  };

  if (typ_is_tentative(t) || !typ_hash_ready(t)) {
    record_tentative(mod, &snap);
  } else {
    record_final(mod, &snap);
  }
}

error topdeps_foreach(struct module *mod, struct node *node,
                      topdeps_each each, void *user) {
  struct toplevel *toplevel = node_toplevel(node);
  if (toplevel->topdeps == NULL) {
    return 0;
  }

  // Need to work when entries are being added to the topdeps while we are
  // iterating over it, so we have to get the current count every time.

  struct vecsnapshot *tentatives = &toplevel->topdeps->tentatives;

  struct top_state *st = mod->state->top_state;
  // 'st' may be NULL when topdeps_foreach() is called from a non-passing
  // context, e.g. cprinter.

  for (ssize_t n = 0; n < vecsnapshot_count(tentatives); ++n) {
    struct snapshot *snap = vecsnapshot_get(tentatives, n);
    struct typ *t = *snap->loc;
    if (st != NULL) {
      if (!typ_is_tentative(t) && typ_hash_ready(t)) {
        record_final(mod, snap);
        n += vecsnapshot_remove_replace_with_last(tentatives, n);
        continue;
      }
    }

    error e = each(mod, node, t, 0, user);
    EXCEPT(e);
  }

  struct vectyp *list = &toplevel->topdeps->list;
{'fraction_non_alphanumeric': 0.13735708174535272, 'fraction_numerical': 0.0035778175313059034, 'mean_word_length': 3.034515218073423, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4849830', 'n_tokens_mistral': 4804, 'n_tokens_neox': 4660, 'n_words': 1078}