<commit_before>/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2004-2016. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#include <stddef.h>	/* offsetof() */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "global.h"
#include <sys/mman.h>
#include "error.h"
#include "bif.h"
#include "big.h"	/* term_to_Sint() */
#include "erl_binary.h"

#include "hipe_arch.h"
#include "hipe_bif0.h"
#include "hipe_native_bif.h"	/* nbif_callemu() */

#undef F_TIMO
#undef THE_NON_VALUE
#undef ERL_FUN_SIZE
#include "hipe_literals.h"

const Uint sse2_fnegate_mask[2] = {0x8000000000000000,0};

void hipe_patch_load_fe(Uint64 *address, Uint64 value)
{
    /* address points to an imm64 operand */
    *address = value;
    hipe_flush_icache_word(address);
}

int hipe_patch_insn(void *address, Uint64 value, Eterm type)
{
    switch (type) {
      case am_closure:
      case am_constant:
      case am_c_const:
	*(Uint64*)address = value;
	break;
      case am_atom:
	/* check that value fits in an unsigned imm32 */
	/* XXX: are we sure it's not really a signed imm32? */
	if ((Uint)(Uint32)value != value)
	    return -1;
	*(Uint32*)address = (Uint32)value;
	break;
      default:
	return -1;
    }
    hipe_flush_icache_word(address);
    return 0;
}

int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
    Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4;

    if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) {
        destOffset = ((Sint64)trampoline - 2) - (Sint64)callAddress - 4;

        if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L))
            return -1;

        *(void**)trampoline = destAddress;
        hipe_flush_icache_word(trampoline);
    }

    *(Uint32*)callAddress = (Uint32)destOffset;
    hipe_flush_icache_word(callAddress);
    return 0;
}

/*
 * Memory allocator for executable code.
 *
 * We use a dedicated allocator for executable code (from OTP 19.0)
 * to make sure the memory we get is executable (PROT_EXEC)
 * and to ensure that executable code ends up in the low 2GB
 * of the address space, as required by HiPE/AMD64's small code model.
 */
static void *alloc_code(unsigned int alloc_bytes)
{
    return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}

static int check_callees(Eterm callees)
{
    Eterm *tuple;
    Uint arity;
    Uint i;

    if (is_not_tuple(callees))
	return -1;
    tuple = tuple_val(callees);
    arity = arityval(tuple[0]);
    for (i = 1; i <= arity; ++i) {
	Eterm mfa = tuple[i];
	if (is_atom(mfa))
	    continue;
	if (is_not_tuple(mfa) ||
	    tuple_val(mfa)[0] != make_arityval(3) ||
	    is_not_atom(tuple_val(mfa)[1]) ||
	    is_not_atom(tuple_val(mfa)[2]) ||
	    is_not_small(tuple_val(mfa)[3]) ||
	    unsigned_val(tuple_val(mfa)[3]) > 255)
	    return -1;
    }
    return arity;
}

#define TRAMPOLINE_BYTES 12

static void generate_trampolines(unsigned char *address,
                                 int nrcallees, Eterm callees,
                                 unsigned char **trampvec)
{
    unsigned char *trampoline = address;
    int i;

    for(i = 0; i < nrcallees; ++i) {
        trampoline[0] = 0x48;           /* movabsq $..., %rax; */
        trampoline[1] = 0xb8;
        *((Uint64*)(trampoline+2)) = 0; /* callee's address */
        trampoline[10] = 0xff;          /* jmpq *%rax */
        trampoline[11] = 0xe0;
        trampvec[i] = trampoline+2;
        trampoline += TRAMPOLINE_BYTES;
    }
    hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES);
}

void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
    int nrcallees;
    Eterm trampvecbin;
    unsigned char **trampvec;
    unsigned char *address;

    nrcallees = check_callees(callees);
    if (nrcallees < 0)
	return NULL;

    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned char*));
    trampvec = (unsigned char **)binary_bytes(trampvecbin);

    address = alloc_code(nrbytes + nrcallees*TRAMPOLINE_BYTES);
    generate_trampolines(address + nrbytes, nrcallees, callees, trampvec);
    *trampolines = trampvecbin;
    return address;
}

void hipe_free_code(void* code, unsigned int bytes)
{
    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
}

/* Make stub for native code calling exported beam function.
*/
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)<commit_msg>erts: Refactor hipe x86_64 trampolines<commit_after>/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2004-2016. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#include <stddef.h>	/* offsetof() */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "global.h"
#include <sys/mman.h>
#include "error.h"
#include "bif.h"
#include "big.h"	/* term_to_Sint() */
#include "erl_binary.h"

#include "hipe_arch.h"
#include "hipe_bif0.h"
#include "hipe_native_bif.h"	/* nbif_callemu() */

#undef F_TIMO
#undef THE_NON_VALUE
#undef ERL_FUN_SIZE
#include "hipe_literals.h"

static void patch_trampoline(void *trampoline, void *destAddress);

const Uint sse2_fnegate_mask[2] = {0x8000000000000000,0};

void hipe_patch_load_fe(Uint64 *address, Uint64 value)
{
    /* address points to an imm64 operand */
    *address = value;
    hipe_flush_icache_word(address);
}

int hipe_patch_insn(void *address, Uint64 value, Eterm type)
{
    switch (type) {
      case am_closure:
      case am_constant:
      case am_c_const:
	*(Uint64*)address = value;
	break;
      case am_atom:
	/* check that value fits in an unsigned imm32 */
	/* XXX: are we sure it's not really a signed imm32? */
	if ((Uint)(Uint32)value != value)
	    return -1;
	*(Uint32*)address = (Uint32)value;
	break;
      default:
	return -1;
    }
    hipe_flush_icache_word(address);
    return 0;
}

int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
    Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4;

    if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) {
        destOffset = (Sint64)trampoline - (Sint64)callAddress - 4;

        if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L))
            return -1;

        patch_trampoline(trampoline, destAddress);
    }

    *(Uint32*)callAddress = (Uint32)destOffset;
    hipe_flush_icache_word(callAddress);
    return 0;
}

/*
 * Memory allocator for executable code.
 *
 * We use a dedicated allocator for executable code (from OTP 19.0)
 * to make sure the memory we get is executable (PROT_EXEC)
 * and to ensure that executable code ends up in the low 2GB
 * of the address space, as required by HiPE/AMD64's small code model.
 */
static void *alloc_code(unsigned int alloc_bytes)
{
    return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}

static int check_callees(Eterm callees)
{
    Eterm *tuple;
    Uint arity;
    Uint i;

    if (is_not_tuple(callees))
	return -1;
    tuple = tuple_val(callees);
    arity = arityval(tuple[0]);
    for (i = 1; i <= arity; ++i) {
	Eterm mfa = tuple[i];
	if (is_atom(mfa))
	    continue;
	if (is_not_tuple(mfa) ||
	    tuple_val(mfa)[0] != make_arityval(3) ||
	    is_not_atom(tuple_val(mfa)[1]) ||
	    is_not_atom(tuple_val(mfa)[2]) ||
	    is_not_small(tuple_val(mfa)[3]) ||
	    unsigned_val(tuple_val(mfa)[3]) > 255)
	    return -1;
    }
    return arity;
}

#define TRAMPOLINE_BYTES 12

static void generate_trampolines(unsigned char *address,
                                 int nrcallees, Eterm callees,
                                 unsigned char **trampvec)
{
    unsigned char *trampoline = address;
    int i;

    for(i = 0; i < nrcallees; ++i) {
        trampoline[0] = 0x48;           /* movabsq $..., %rax; */
        trampoline[1] = 0xb8;
        *(void**)(trampoline+2) = NULL; /* callee's address */
        trampoline[10] = 0xff;          /* jmpq *%rax */
        trampoline[11] = 0xe0;
        trampvec[i] = trampoline;
        trampoline += TRAMPOLINE_BYTES;
    }
    hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES);
}

static void patch_trampoline(void *trampoline, void *destAddress)
{
    unsigned char *tp = (unsigned char*) trampoline;

    ASSERT(tp[0] == 0x48 && tp[1] == 0xb8);

    *(void**)(tp+2) = destAddress; /* callee's address */
    hipe_flush_icache_word(tp+2);
}

void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
    int nrcallees;
    Eterm trampvecbin;
    unsigned char **trampvec;
    unsigned char *address;

    nrcallees = check_callees(callees);
    if (nrcallees < 0)
	return NULL;

    trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned char*));
    trampvec = (unsigned char **)binary_bytes(trampvecbin);

    address = alloc_code(nrbytes + nrcallees*TRAMPOLINE_BYTES);
    generate_trampolines(address + nrbytes, nrcallees, callees, trampvec);
    *trampolines = trampvecbin;
    return address;
}

void hipe_free_code(void* code, unsigned int bytes)
{
    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
}

/* Make stub for native code calling exported beam function.
*/
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{'fraction_non_alphanumeric': 0.11765285025105839, 'fraction_numerical': 0.030914640149650487, 'mean_word_length': 3.412684622067767, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5276739', 'n_tokens_mistral': 3868, 'n_tokens_neox': 3591, 'n_words': 926}