<commit_before>/*
** ARM64 IR assembler (SSA IR -> machine code).
** Copyright (C) 2005-2017 <NAME>. See Copyright Notice in luajit.h
**
** Contributed by <NAME> and <NAME> from RT-RK.com.
** Sponsored by Cisco Systems, Inc.
*/

/* -- Register allocator extensions --------------------------------------- */

/* Allocate a register with a hint. */
static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
{
  Reg r = IR(ref)->r;
  if (ra_noreg(r)) {
    if (!ra_hashint(r) && !iscrossref(as, ref))
      ra_sethint(IR(ref)->r, hint);  /* Propagate register hint. */
    r = ra_allocref(as, ref, allow);
  }
  ra_noweak(as, r);
  return r;
}

/* Allocate two source registers for three-operand instructions. */
static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
{
  IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
  Reg left = irl->r, right = irr->r;
  if (ra_hasreg(left)) {
    ra_noweak(as, left);
    if (ra_noreg(right))
      right = ra_allocref(as, ir->op2, rset_exclude(allow, left));
    else
      ra_noweak(as, right);
  } else if (ra_hasreg(right)) {
    ra_noweak(as, right);
    left = ra_allocref(as, ir->op1, rset_exclude(allow, right));
  } else if (ra_hashint(right)) {
    right = ra_allocref(as, ir->op2, allow);
    left = ra_alloc1(as, ir->op1, rset_exclude(allow, right));
  } else {
    left = ra_allocref(as, ir->op1, allow);
    right = ra_alloc1(as, ir->op2, rset_exclude(allow, left));
  }
  return left | (right << 8);
}

/* -- Guard handling ------------------------------------------------------ */

/* Setup all needed exit stubs. */
static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
{
  ExitNo i;
  MCode *mxp = as->mctop;
  if (mxp - (nexits + 3 + MCLIM_REDZONE) < as->mclim)
    asm_mclimit(as);
  /* 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno; bl <1; bl <1; ... */
  for (i = nexits-1; (int32_t)i >= 0; i--)
    *--mxp = A64I_BL|((-3-i)&0x03ffffffu);
  *--mxp = A64I_MOVZw|A64F_U16(as->T->traceno);
  mxp--;
  *mxp = A64I_BL|(((MCode *)(void *)lj_vm_exit_handler-mxp)&0x03ffffffu);
  *--mxp = A64I_STRx|A64F_D(RID_LR)|A64F_N(RID_SP);
  as->mctop = mxp;
}

static MCode *asm_exitstub_addr(ASMState *as, ExitNo exitno)
{
  /* Keep this in-sync with exitstub_trace_addr(). */
  return as->mctop + exitno + 3;
}

/* Emit conditional branch to exit for guard. */
static void asm_guardcc(ASMState *as, A64CC cc)
{
  MCode *target = asm_exitstub_addr(as, as->snapno);
  MCode *p = as->mcp;
  if (LJ_UNLIKELY(p == as->invmcp)) {
    as->loopinv = 1;
    *p = A64I_B | ((target-p) & 0x03ffffffu);
    emit_cond_branch(as, cc^1, p-1);
    return;
  }
  emit_cond_branch(as, cc, target);
}

/* Emit test and branch instruction to exit for guard. */
static void asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit)
{
  MCode *target = asm_exitstub_addr(as, as->snapno);
  MCode *p = as->mcp;
  if (LJ_UNLIKELY(p == as->invmcp)) {
    as->loopinv = 1;
    *p = A64I_B | ((target-p) & 0x03ffffffu);
    emit_tnb(as, ai^0x01000000u, r, bit, p-1);
    return;
  }
  emit_tnb(as, ai, r, bit, target);
}

/* Emit compare and branch instruction to exit for guard. */
static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r)
{
  MCode *target = asm_exitstub_addr(as, as->snapno);
  MCode *p = as->mcp;
  if (LJ_UNLIKELY(p == as->invmcp)) {
    as->loopinv = 1;
    *p = A64I_B | ((target-p) & 0x03ffffffu);
    emit_cnb(as, ai^0x01000000u, r, p-1);
    return;
  }
  emit_cnb(as, ai, r, target);
}

/* -- Operand fusion ------------------------------------------------------ */

/* Limit linear search to this distance. Avoids O(n^2) behavior. */
#define CONFLICT_SEARCH_LIM	31

static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
{
  if (irref_isk(ref)) {
    IRIns *ir = IR(ref);
    if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
      *k = ir->i;
      return 1;
    } else if (checki32((int64_t)ir_k64(ir)->u64)) {
      *k = (int32_t)ir_k64(ir)->u64;
      return 1;
    }
  }
  return 0;
}

/* Check if there's no conflicting instruction between curins and ref. */
static int noconflict(ASMState *as, IRRef ref, IROp conflict)
{
  IRIns *ir = as->ir;
  IRRef i = as->curins;
  if (i > ref + CONFLICT_SEARCH_LIM)
    return 0;  /* Give up, ref is too far away. */
  while (--i > ref)
    if (ir[i].o == conflict)
      return 0;  /* Conflict found. */
  return 1;  /* Ok, no conflict. */
}

/* Fuse the array base of colocated arrays. */
static int32_t asm_fuseabase(ASMState *as, IRRef ref)
{
  IRIns *ir = IR(ref);
  if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
      !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
    return (int32_t)sizeof(GCtab);
  return 0;
}

#define FUSE_REG	0x40000000

/* Fuse array/hash/upvalue reference into register+offset operand. */
static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow,
			  A64Ins ins)
{
  IRIns *ir = IR(ref);
  if (ra_noreg(ir->r)) {
    if (ir->o == IR_AREF) {
      if (mayfuse(as, ref)) {
	if (irref_isk(ir->op2)) {
	  IRRef tab = IR(ir->op1)->op1;
	  int32_t ofs = asm_fuseabase(as, tab);
	  IRRef refa = ofs ? tab : ir->op1;
	  ofs += 8*IR(ir->op2)->i;
	  if (emit_checkofs(ins, ofs)) {
	    *ofsp = ofs;
	    return ra_alloc1(as, refa, allow);
	  }
	} else {
	  Reg base = ra_alloc1(as, ir->op1, allow);
	  *ofsp = FUSE_REG|ra_alloc1(as, ir->op2, rset_exclude(allow, base));
	  return base;
	}
      }
    } else if (ir->o == IR_HREFK) {
      if (mayfuse(as, ref)) {
	int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
	if (emit_checkofs(ins, ofs)) {
	  *ofsp = ofs;
	  return ra_alloc1(as, ir->op1, allow);
	}
      }
    } else if (ir->o == IR_UREFC) {
      if (irref_isk(ir->op1)) {
	GCfunc *fn = ir_kfunc(IR(ir->op1));
	GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
	int64_t ofs = glofs(as, &uv->tv);
	if (emit_checkofs(ins, ofs)) {
	  *ofsp = (int32_t)ofs;
	  return RID_GL;
	}
      }
    }
  }
  *ofsp = 0;
  return ra_alloc1(as, ref, allow);
}

/* Fuse m operand into arithmetic/logic instructions. */
static uint32_t asm_fuseopm(ASMState *as, A64Ins ai, IRRef ref, RegSet allow)
{
  IRIns *ir = IR(ref);
  if (ra_hasreg(ir->r)) {
    ra_noweak(as, ir->r);
    return A64F_M(ir->r);
  } else if (irref_isk(ref)) {
    uint32_t m;
    int64_t k = get_k64val(ir);
    if ((ai & 0x1f000000) == 0x0a000000)
      m = emit_isk13(k, irt_is64(ir->t));
    else
      m = emit_isk12(k);
    if (m)
      return m;
  } else if (mayfuse(as, ref)) {
    if ((ir->o >= IR_BSHL && ir->o <= IR_BSAR && irref_isk(ir->op2)) ||
	(ir->o == IR_ADD && ir->op1 == ir->op2)) {
      A64Shift sh = ir->o == IR_BSHR ? A64SH_LSR :
		    ir->o == IR_BSAR ? A64SH_ASR : A64SH_LSL;
      int shift = ir->o == IR_ADD ? 1 :
		    (IR(ir->op2)->i & (irt_is64(ir->t) ? 63 : 31));
      IRIns *irl = IR(ir->op1);
      if (sh == A64SH_LSL &&
	  irl->o == IR_CONV &&
	  irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
	  shift <= 4 &&
	  canfuse(as, irl)) {
	Reg m = ra_alloc1(as, irl->op1, allow);
	return A64F_M(m) | A64F_EXSH(A64EX_SXTW, shift);
      } else {
	Reg m = ra_alloc1(as, ir->op1, allow);
	return A64F_M(m) | A64F_SH(sh, shift);
      }
    } else if (ir->o == IR_CONV &&
	       ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)) {
      Reg m = ra_alloc1(as, ir->op1, allow);
      return A64F_M(m) | A64F_EX(A64EX_SXTW);
    }
  }
  return A64F_M(ra_allocref(as, ref, allow));
}

/* Fuse XLOAD/XSTORE reference into load/store operand. */
static void asm_fusexref(ASMState *as, A64Ins ai, Reg rd, IRRef ref,
			 RegSet allow)
{
  IRIns *ir = IR(ref);
  Reg base;
  int32_t ofs = 0;
  if (ra_noreg(ir->r) && canfuse(as, ir)) {
    if (ir->o == IR_ADD) {
      if (asm_isk32(as, ir->op2, &ofs) && emit_checkofs(ai, ofs)) {
	ref = ir->op1;
      } else {
	Reg rn, rm;
	IRRef lref = ir->op1, rref = ir->op2;
	IRIns *irl = IR(lref);
	if (mayfuse(as, irl->op1)) {
	  unsigned int shift = 4;
	  if (irl->o == IR_BSHL && irref_isk(irl->op2)) {
	    shift = (IR(irl->op2)->i & 63);
	  } else if (irl->o == IR_ADD && irl->op1 == irl->op2) {
	    shift = 1;
	  }
	  if ((ai >> 30) == shift) {
	    lref = irl->op1;
	    irl = IR(lref);
	    ai |= A64I_LS_SH;
	  }
	}
	if (irl->o == IR_CONV &&
	    irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
	    canfuse(as, irl)) {
	  lref = irl->op1;
	  ai |= A64I_LS_SXTWx;
	} else {
	  ai |= A64I_LS_LSLx;
	}
	rm = ra_alloc1(as, lref, allow);
	rn = ra_alloc1(as, rref, rset_exclude(allow, rm));
	emit_dnm(as, (ai^A64I_LS_R), rd, rn, rm);
	return;
      }<commit_msg>ARM64: Fix XLOAD/XSTORE with FP operand.<commit_after>/*
** ARM64 IR assembler (SSA IR -> machine code).
** Copyright (C) 2005-2017 <NAME>. See Copyright Notice in luajit.h
**
** Contributed by <NAME> and <NAME> from RT-RK.com.
** Sponsored by Cisco Systems, Inc.
*/

/* -- Register allocator extensions --------------------------------------- */

/* Allocate a register with a hint. */
static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
{
  Reg r = IR(ref)->r;
  if (ra_noreg(r)) {
    if (!ra_hashint(r) && !iscrossref(as, ref))
      ra_sethint(IR(ref)->r, hint);  /* Propagate register hint. */
    r = ra_allocref(as, ref, allow);
  }
  ra_noweak(as, r);
  return r;
}

/* Allocate two source registers for three-operand instructions. */
static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
{
  IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
  Reg left = irl->r, right = irr->r;
  if (ra_hasreg(left)) {
    ra_noweak(as, left);
    if (ra_noreg(right))
      right = ra_allocref(as, ir->op2, rset_exclude(allow, left));
    else
      ra_noweak(as, right);
  } else if (ra_hasreg(right)) {
    ra_noweak(as, right);
    left = ra_allocref(as, ir->op1, rset_exclude(allow, right));
  } else if (ra_hashint(right)) {
    right = ra_allocref(as, ir->op2, allow);
    left = ra_alloc1(as, ir->op1, rset_exclude(allow, right));
  } else {
    left = ra_allocref(as, ir->op1, allow);
    right = ra_alloc1(as, ir->op2, rset_exclude(allow, left));
  }
  return left | (right << 8);
}

/* -- Guard handling ------------------------------------------------------ */

/* Setup all needed exit stubs. */
static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
{
  ExitNo i;
  MCode *mxp = as->mctop;
  if (mxp - (nexits + 3 + MCLIM_REDZONE) < as->mclim)
    asm_mclimit(as);
  /* 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno; bl <1; bl <1; ... */
  for (i = nexits-1; (int32_t)i >= 0; i--)
    *--mxp = A64I_BL|((-3-i)&0x03ffffffu);
  *--mxp = A64I_MOVZw|A64F_U16(as->T->traceno);
  mxp--;
  *mxp = A64I_BL|(((MCode *)(void *)lj_vm_exit_handler-mxp)&0x03ffffffu);
  *--mxp = A64I_STRx|A64F_D(RID_LR)|A64F_N(RID_SP);
  as->mctop = mxp;
}

static MCode *asm_exitstub_addr(ASMState *as, ExitNo exitno)
{
  /* Keep this in-sync with exitstub_trace_addr(). */
  return as->mctop + exitno + 3;
}

/* Emit conditional branch to exit for guard. */
static void asm_guardcc(ASMState *as, A64CC cc)
{
  MCode *target = asm_exitstub_addr(as, as->snapno);
  MCode *p = as->mcp;
  if (LJ_UNLIKELY(p == as->invmcp)) {
    as->loopinv = 1;
    *p = A64I_B | ((target-p) & 0x03ffffffu);
    emit_cond_branch(as, cc^1, p-1);
    return;
  }
  emit_cond_branch(as, cc, target);
}

/* Emit test and branch instruction to exit for guard. */
static void asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit)
{
  MCode *target = asm_exitstub_addr(as, as->snapno);
  MCode *p = as->mcp;
  if (LJ_UNLIKELY(p == as->invmcp)) {
    as->loopinv = 1;
    *p = A64I_B | ((target-p) & 0x03ffffffu);
    emit_tnb(as, ai^0x01000000u, r, bit, p-1);
    return;
  }
  emit_tnb(as, ai, r, bit, target);
}

/* Emit compare and branch instruction to exit for guard. */
static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r)
{
  MCode *target = asm_exitstub_addr(as, as->snapno);
  MCode *p = as->mcp;
  if (LJ_UNLIKELY(p == as->invmcp)) {
    as->loopinv = 1;
    *p = A64I_B | ((target-p) & 0x03ffffffu);
    emit_cnb(as, ai^0x01000000u, r, p-1);
    return;
  }
  emit_cnb(as, ai, r, target);
}

/* -- Operand fusion ------------------------------------------------------ */

/* Limit linear search to this distance. Avoids O(n^2) behavior. */
#define CONFLICT_SEARCH_LIM	31

static int asm_isk32(ASMState *as, IRRef ref, int32_t *k)
{
  if (irref_isk(ref)) {
    IRIns *ir = IR(ref);
    if (ir->o == IR_KNULL || !irt_is64(ir->t)) {
      *k = ir->i;
      return 1;
    } else if (checki32((int64_t)ir_k64(ir)->u64)) {
      *k = (int32_t)ir_k64(ir)->u64;
      return 1;
    }
  }
  return 0;
}

/* Check if there's no conflicting instruction between curins and ref. */
static int noconflict(ASMState *as, IRRef ref, IROp conflict)
{
  IRIns *ir = as->ir;
  IRRef i = as->curins;
  if (i > ref + CONFLICT_SEARCH_LIM)
    return 0;  /* Give up, ref is too far away. */
  while (--i > ref)
    if (ir[i].o == conflict)
      return 0;  /* Conflict found. */
  return 1;  /* Ok, no conflict. */
}

/* Fuse the array base of colocated arrays. */
static int32_t asm_fuseabase(ASMState *as, IRRef ref)
{
  IRIns *ir = IR(ref);
  if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
      !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
    return (int32_t)sizeof(GCtab);
  return 0;
}

#define FUSE_REG	0x40000000

/* Fuse array/hash/upvalue reference into register+offset operand. */
static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow,
			  A64Ins ins)
{
  IRIns *ir = IR(ref);
  if (ra_noreg(ir->r)) {
    if (ir->o == IR_AREF) {
      if (mayfuse(as, ref)) {
	if (irref_isk(ir->op2)) {
	  IRRef tab = IR(ir->op1)->op1;
	  int32_t ofs = asm_fuseabase(as, tab);
	  IRRef refa = ofs ? tab : ir->op1;
	  ofs += 8*IR(ir->op2)->i;
	  if (emit_checkofs(ins, ofs)) {
	    *ofsp = ofs;
	    return ra_alloc1(as, refa, allow);
	  }
	} else {
	  Reg base = ra_alloc1(as, ir->op1, allow);
	  *ofsp = FUSE_REG|ra_alloc1(as, ir->op2, rset_exclude(allow, base));
	  return base;
	}
      }
    } else if (ir->o == IR_HREFK) {
      if (mayfuse(as, ref)) {
	int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
	if (emit_checkofs(ins, ofs)) {
	  *ofsp = ofs;
	  return ra_alloc1(as, ir->op1, allow);
	}
      }
    } else if (ir->o == IR_UREFC) {
      if (irref_isk(ir->op1)) {
	GCfunc *fn = ir_kfunc(IR(ir->op1));
	GCupval *uv = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv;
	int64_t ofs = glofs(as, &uv->tv);
	if (emit_checkofs(ins, ofs)) {
	  *ofsp = (int32_t)ofs;
	  return RID_GL;
	}
      }
    }
  }
  *ofsp = 0;
  return ra_alloc1(as, ref, allow);
}

/* Fuse m operand into arithmetic/logic instructions. */
static uint32_t asm_fuseopm(ASMState *as, A64Ins ai, IRRef ref, RegSet allow)
{
  IRIns *ir = IR(ref);
  if (ra_hasreg(ir->r)) {
    ra_noweak(as, ir->r);
    return A64F_M(ir->r);
  } else if (irref_isk(ref)) {
    uint32_t m;
    int64_t k = get_k64val(ir);
    if ((ai & 0x1f000000) == 0x0a000000)
      m = emit_isk13(k, irt_is64(ir->t));
    else
      m = emit_isk12(k);
    if (m)
      return m;
  } else if (mayfuse(as, ref)) {
    if ((ir->o >= IR_BSHL && ir->o <= IR_BSAR && irref_isk(ir->op2)) ||
	(ir->o == IR_ADD && ir->op1 == ir->op2)) {
      A64Shift sh = ir->o == IR_BSHR ? A64SH_LSR :
		    ir->o == IR_BSAR ? A64SH_ASR : A64SH_LSL;
      int shift = ir->o == IR_ADD ? 1 :
		    (IR(ir->op2)->i & (irt_is64(ir->t) ? 63 : 31));
      IRIns *irl = IR(ir->op1);
      if (sh == A64SH_LSL &&
	  irl->o == IR_CONV &&
	  irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
	  shift <= 4 &&
	  canfuse(as, irl)) {
	Reg m = ra_alloc1(as, irl->op1, allow);
	return A64F_M(m) | A64F_EXSH(A64EX_SXTW, shift);
      } else {
	Reg m = ra_alloc1(as, ir->op1, allow);
	return A64F_M(m) | A64F_SH(sh, shift);
      }
    } else if (ir->o == IR_CONV &&
	       ir->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT)) {
      Reg m = ra_alloc1(as, ir->op1, allow);
      return A64F_M(m) | A64F_EX(A64EX_SXTW);
    }
  }
  return A64F_M(ra_allocref(as, ref, allow));
}

/* Fuse XLOAD/XSTORE reference into load/store operand. */
static void asm_fusexref(ASMState *as, A64Ins ai, Reg rd, IRRef ref,
			 RegSet allow)
{
  IRIns *ir = IR(ref);
  Reg base;
  int32_t ofs = 0;
  if (ra_noreg(ir->r) && canfuse(as, ir)) {
    if (ir->o == IR_ADD) {
      if (asm_isk32(as, ir->op2, &ofs) && emit_checkofs(ai, ofs)) {
	ref = ir->op1;
      } else {
	Reg rn, rm;
	IRRef lref = ir->op1, rref = ir->op2;
	IRIns *irl = IR(lref);
	if (mayfuse(as, irl->op1)) {
	  unsigned int shift = 4;
	  if (irl->o == IR_BSHL && irref_isk(irl->op2)) {
	    shift = (IR(irl->op2)->i & 63);
	  } else if (irl->o == IR_ADD && irl->op1 == irl->op2) {
	    shift = 1;
	  }
	  if ((ai >> 30) == shift) {
	    lref = irl->op1;
	    irl = IR(lref);
	    ai |= A64I_LS_SH;
	  }
	}
	if (irl->o == IR_CONV &&
	    irl->op2 == ((IRT_I64<<IRCONV_DSH)|IRT_INT|IRCONV_SEXT) &&
	    canfuse(as, irl)) {
	  lref = irl->op1;
	  ai |= A64I_LS_SXTWx;
	} else {
	  ai |= A64I_LS_LSLx;
	}
	rm = ra_alloc1(as, lref, allow);
	rn = ra_alloc1(as, rref, rset_exclude(allow, rm));
	emit_dnm(as, (ai^A64I_LS_R), (rd & 31), rn, rm);
	return;
      }
{'fraction_non_alphanumeric': 0.19347647541945323, 'fraction_numerical': 0.036137510266338144, 'mean_word_length': 2.8446098331078034, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 233, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7443002', 'n_tokens_mistral': 7939, 'n_tokens_neox': 7435, 'n_words': 1662}