diff options
Diffstat (limited to 'erts/emulator/hipe')
44 files changed, 1697 insertions, 2196 deletions
diff --git a/erts/emulator/hipe/elf64ppc.x b/erts/emulator/hipe/elf64ppc.x index 46d2632970..f9b3e6795d 100644 --- a/erts/emulator/hipe/elf64ppc.x +++ b/erts/emulator/hipe/elf64ppc.x @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2016. All Rights Reserved. + * Copyright Ericsson AB 2005-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ SEARCH_DIR("/mnt/archive/cross-ppc64/ppc64-unknown-linux/lib"); SECTIONS { /* Read-only sections, merged into text segment: */ - PROVIDE (__executable_start = 0x0180000); . = 0x01800000 + SIZEOF_HEADERS; + PROVIDE (__executable_start = 0x01800000); . = 0x01800000 + SIZEOF_HEADERS; .interp : { *(.interp) } .hash : { *(.hash) } .dynsym : { *(.dynsym) } diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..e3cff4a4ba 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Sint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Sint)destAddress - (Sint)callAddress - 4; if ((Sint)(Sint32)rel32 != rel32) return -1; @@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - /* * Memory allocator for executable code. * @@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void) */ static void *alloc_code(unsigned int alloc_bytes) { - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } @@ -130,6 +104,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +213,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_amd64_asm.m4 b/erts/emulator/hipe/hipe_amd64_asm.m4 index 2c0fbbee2d..409fd0ef89 100644 --- a/erts/emulator/hipe/hipe_amd64_asm.m4 +++ b/erts/emulator/hipe/hipe_amd64_asm.m4 @@ -121,6 +121,22 @@ define(NSP,%rsp)dnl /* + * Debugging macros + * + * Keeps track of whether context has been saved in the debug build, allowing us + * to detect when the garbage collector is called when it shouldn't. + */ +`#ifdef DEBUG +# define SET_GC_UNSAFE \ + movq $1, P_GCUNSAFE(P) +# define SET_GC_SAFE \ + movq $0, P_GCUNSAFE(P) +#else +# define SET_GC_UNSAFE +# define SET_GC_SAFE +#endif' + +/* * Context switching macros. */ `#define SWITCH_C_TO_ERLANG_QUICK \ @@ -133,12 +149,14 @@ define(NSP,%rsp)dnl `#define SAVE_CACHED_STATE \ SAVE_HP; \ - SAVE_FCALLS' + SAVE_FCALLS; \ + SET_GC_SAFE' `#define RESTORE_CACHED_STATE \ RESTORE_HP; \ RESTORE_HEAP_LIMIT; \ - RESTORE_FCALLS' + RESTORE_FCALLS; \ + SET_GC_UNSAFE' `#define SWITCH_C_TO_ERLANG \ RESTORE_CACHED_STATE; \ diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4 index 9cf3bf74fd..dca3887564 100644 --- a/erts/emulator/hipe/hipe_amd64_bifs.m4 +++ b/erts/emulator/hipe/hipe_amd64_bifs.m4 @@ -41,11 +41,11 @@ define(HANDLE_GOT_MBUF,` `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) # define CALL_BIF(F) \ - movq CSYM(F)@GOTPCREL(%rip), %r11; \ + movq CSYM(nbif_impl_##F)@GOTPCREL(%rip), %r11; \ movq %r11, P_BIF_CALLEE(P); \ call CSYM(hipe_debug_bif_wrapper) #else -# define CALL_BIF(F) call CSYM(F) +# define CALL_BIF(F) call CSYM(nbif_impl_##F) #endif' /* @@ -595,15 +595,12 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu) #endif /* NO_FPE_SIGNALS */ /* - * Implement gc_bif_interface_0 as nofail_primop_interface_0. - */ -define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)') - -/* - * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2). + * Implement gc_bif_interface_N as standard_bif_interface_N. */ +define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)') define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)') define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)') +define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)') /* * Implement gc_nofail_primop_interface_1 as nofail_primop_interface_1. diff --git a/erts/emulator/hipe/hipe_amd64_glue.S b/erts/emulator/hipe/hipe_amd64_glue.S index b37ed3c68a..f3404888d5 100644 --- a/erts/emulator/hipe/hipe_amd64_glue.S +++ b/erts/emulator/hipe/hipe_amd64_glue.S @@ -94,6 +94,7 @@ ASYM(nbif_return): .nosave_exit: /* switch to C stack */ SWITCH_ERLANG_TO_C_QUICK + SET_GC_SAFE /* restore C callee-save registers, drop frame, return */ movq (%rsp), %rbp # kills P movq 8(%rsp), %rbx @@ -398,6 +399,7 @@ nbif_4_simple_exception: movl %eax, P_NARITY(P) # Note: narity is a 32-bit field /* find and prepare to invoke the handler */ SWITCH_ERLANG_TO_C_QUICK # The cached state is clean and need not be saved. + SET_GC_SAFE movq P, %rdi call CSYM(hipe_handle_exception) # Note: hipe_handle_exception() conses SWITCH_C_TO_ERLANG # %rsp updated by hipe_find_handler() diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..059b8e7f29 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -30,23 +30,31 @@ extern void hipe_patch_load_fe(Uint *address, Uint value); extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); -extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p); +extern void hipe_free_code(void*, unsigned int); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..b61939724c 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address) hipe_flush_icache_range(address, 4); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ - /* On ARM we always allocate a trampoline at base+32MB-8 for - nbif_callemu, so tramp_pos <= base+32MB-8. */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} static int check_callees(Eterm callees) { @@ -107,126 +82,53 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) -{ - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; - Eterm mfa, m, f; - unsigned int a, *trampoline; - - m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */ - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; +#define TRAMPOLINE_WORDS 2 - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - mfa = tuple_val(callees)[trampnr]; - if (is_atom(mfa)) - trampoline = hipe_primop_get_trampoline(mfa); - else { - m = tuple_val(mfa)[1]; - f = tuple_val(mfa)[2]; - a = unsigned_val(tuple_val(mfa)[3]); - trampoline = hipe_mfa_get_trampoline(m, f, a); - } - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { - if (nrfreewords < 2) - return NULL; - nrfreewords -= 2; - tramp_pos = trampoline = tramp_pos - 2; - trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - trampoline[1] = 0; /* callee's address */ - hipe_flush_icache_range(trampoline, 2*sizeof(int)); - if (is_atom(mfa)) - hipe_primop_set_trampoline(mfa, trampoline); - else - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) +{ + Uint32* trampoline = address; + int i; + + for (i = 0; i < nrcallees; ++i) { + trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ + trampoline[1] = 0; /* callee's address */ + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2; - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } /* @@ -266,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; - unsigned int *tramp_callemu; int callemu_offset; + int is_short_jmp; /* * Native code calls BEAM via a stub looking as follows: @@ -277,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) * b nbif_callemu * .long callee_exp * + * or if nbif_callemu is too far away: + * + * mov r0, #beamArity + * ldr r8, [pc,#0] // callee_exp + * ldr pc, [pc,#0] // nbif_callemu + * .long callee_exp + * .long nbif_callemu + * * I'm using r0 and r8 since they aren't used for - * parameter passing in native code. The branch to - * nbif_callemu may need to go via a trampoline. - * (Trampolines are allowed to modify r12, but they don't.) + * parameter passing in native code. */ - code = alloc_stub(4, &tramp_callemu); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32)); if (!code) return NULL; callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) { - callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) - abort(); + is_short_jmp = (callemu_offset >= -0x00800000 && + callemu_offset <= 0x007FFFFF); +#ifdef DEBUG + if (is_short_jmp && (callemu_offset % 3)==0) { + is_short_jmp = 0; } +#endif /* mov r0, #beamArity */ code[0] = 0xE3A00000 | (beamArity & 0xFF); /* ldr r8, [pc,#0] // callee_exp */ code[1] = 0xE59F8000; - /* b nbif_callemu */ - code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + if (is_short_jmp) { + /* b nbif_callemu */ + code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + } + else { + /* ldr pc, [pc,#0] // nbif_callemu */ + code[2] = 0xE59FF000; + /* .long nbif_callemu */ + code[4] = (unsigned int)&nbif_callemu; + } /* .long callee_exp */ code[3] = (unsigned int)callee_exp; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 5*sizeof(Uint32)); return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA) { Uint32 oldI = *address; diff --git a/erts/emulator/hipe/hipe_arm_asm.m4 b/erts/emulator/hipe/hipe_arm_asm.m4 index ae9ec752bb..68a6faa70b 100644 --- a/erts/emulator/hipe/hipe_arm_asm.m4 +++ b/erts/emulator/hipe/hipe_arm_asm.m4 @@ -48,6 +48,24 @@ define(NR_ARG_REGS,3)dnl admissible values are 0 to 6, inclusive `#define TEMP_LR r8' /* + * Debugging macros + * + * Keeps track of whether context has been saved in the debug build, allowing us + * to detect when the garbage collector is called when it shouldn't. + */ +`#ifdef DEBUG +# define SET_GC_UNSAFE(SCRATCH) \ + mov SCRATCH, #1; \ + str SCRATCH, [P, #P_GCUNSAFE] +# define SET_GC_SAFE(SCRATCH) \ + mov SCRATCH, #0; \ + str SCRATCH, [P, #P_GCUNSAFE] +#else +# define SET_GC_UNSAFE(SCRATCH) +# define SET_GC_SAFE(SCRATCH) +#endif' + +/* * Context switching macros. * * RESTORE_CONTEXT and RESTORE_CONTEXT_QUICK do not affect @@ -59,12 +77,14 @@ define(NR_ARG_REGS,3)dnl admissible values are 0 to 6, inclusive `#define RESTORE_CONTEXT_QUICK \ mov lr, TEMP_LR' -`#define SAVE_CACHED_STATE \ - str HP, [P, #P_HP]; \ - str NSP, [P, #P_NSP]' +`#define SAVE_CACHED_STATE \ + str HP, [P, #P_HP]; \ + str NSP, [P, #P_NSP]; \ + SET_GC_SAFE(HP)' -`#define RESTORE_CACHED_STATE \ - ldr HP, [P, #P_HP]; \ +`#define RESTORE_CACHED_STATE \ + SET_GC_UNSAFE(HP); \ + ldr HP, [P, #P_HP]; \ ldr NSP, [P, #P_NSP]' `#define SAVE_CONTEXT_BIF \ @@ -75,12 +95,14 @@ define(NR_ARG_REGS,3)dnl admissible values are 0 to 6, inclusive ldr HP, [P, #P_HP]' `#define SAVE_CONTEXT_GC \ + SET_GC_SAFE(TEMP_LR); \ mov TEMP_LR, lr; \ str lr, [P, #P_NRA]; \ str NSP, [P, #P_NSP]; \ str HP, [P, #P_HP]' `#define RESTORE_CONTEXT_GC \ + SET_GC_UNSAFE(HP); \ ldr HP, [P, #P_HP]' /* diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4 index d9c9952dbf..a9097dabde 100644 --- a/erts/emulator/hipe/hipe_arm_bifs.m4 +++ b/erts/emulator/hipe/hipe_arm_bifs.m4 @@ -30,9 +30,9 @@ include(`hipe/hipe_arm_asm.m4') .arm `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) ldr r14, =F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper +# define CALL_BIF(F) ldr r14, =nbif_impl_##F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper #else -# define CALL_BIF(F) bl F +# define CALL_BIF(F) bl nbif_impl_##F #endif' define(TEST_GOT_MBUF,`ldr r1, [P, #P_MBUF] /* `TEST_GOT_MBUF' */ @@ -198,8 +198,9 @@ $1: * gc_bif_interface_0(nbif_name, cbif_name) * gc_bif_interface_1(nbif_name, cbif_name) * gc_bif_interface_2(nbif_name, cbif_name) + * gc_bif_interface_3(nbif_name, cbif_name) * - * Generate native interface for a BIF with 0-2 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. * The BIF may do a GC. */ @@ -279,6 +280,36 @@ $1: .type $1, %function #endif') +define(gc_bif_interface_3, +` +#ifndef HAVE_$1 +#`define' HAVE_$1 + .global $1 +$1: + /* Set up C argument registers. */ + mov r0, P + NBIF_ARG(r1,3,0) + NBIF_ARG(r2,3,1) + NBIF_ARG(r3,3,2) + + /* Save caller-save registers and call the C function. */ + SAVE_CONTEXT_GC + str r1, [r0, #P_ARG0] /* Store BIF__ARGS in def_arg_reg[] */ + str r2, [r0, #P_ARG1] + str r3, [r0, #P_ARG2] + add r1, r0, #P_ARG0 + CALL_BIF($2) + TEST_GOT_MBUF(3) + + /* Restore registers. Check for exception. */ + cmp r0, #THE_NON_VALUE + RESTORE_CONTEXT_GC + beq nbif_3_simple_exception + NBIF_RET(3) + .size $1, .-$1 + .type $1, %function +#endif') + /* * gc_nofail_primop_interface_1(nbif_name, cbif_name) * diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S index 49ffa8b1d8..5b7f8ad52d 100644 --- a/erts/emulator/hipe/hipe_arm_glue.S +++ b/erts/emulator/hipe/hipe_arm_glue.S @@ -342,6 +342,7 @@ nbif_4_gc_after_bif: str r1, [P, #P_NARITY] str TEMP_LR, [P, #P_NRA] str NSP, [P, #P_NSP] + SET_GC_SAFE(TEMP_LR) mov TEMP_LR, lr mov r3, #0 /* Pass 0 in arity */ mov r2, #0 /* Pass NULL in regs */ @@ -349,6 +350,7 @@ nbif_4_gc_after_bif: mov r0, P bl erts_gc_after_bif_call mov lr, TEMP_LR + SET_GC_UNSAFE(TEMP_LR) ldr TEMP_LR, [P, #P_NRA] mov r1, #0 str r1, [P, #P_NARITY] @@ -404,6 +406,7 @@ nbif_4_simple_exception: str NSP, [P, #P_NSP] str TEMP_LR, [P, #P_NRA] str r1, [P, #P_NARITY] + SET_GC_SAFE(r0) /* find and prepare to invoke the handler */ mov r0, P bl hipe_handle_exception /* Note: hipe_handle_exception() conses */ @@ -423,6 +426,7 @@ nbif_4_simple_exception: str NSP, [P, #P_NSP] str r1, [P, #P_NARITY] str TEMP_LR, [P, #P_NRA] + SET_GC_SAFE(NSP) b .nosave_exit /* diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 3336fded7a..0225f17613 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,7 @@ #include "hipe_mode_switch.h" #include "hipe_native_bif.h" #include "hipe_bif0.h" +#include "hipe_load.h" /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines a few constants. #undef them here to avoid warnings. */ #undef F_TIMO @@ -54,6 +55,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -374,15 +376,28 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2) } /* + * BIFs for loading code. + */ + +static HipeLoaderState *get_loader_state(Eterm term) +{ + if (!is_internal_magic_ref(term)) return NULL; + + return hipe_get_loader_state(erts_magic_ref2bin(term)); +} + + +/* * Allocate memory and copy machine code to it. */ -BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) { Uint nrbytes; void *bytes; void *address; Eterm trampolines; Eterm *hp; + HipeLoaderState *stp; #ifndef DEBUG ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); @@ -391,7 +406,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint bitsize; #endif - if (is_not_binary(BIF_ARG_1)) + if (is_not_binary(BIF_ARG_1) || + (!(stp = get_loader_state(BIF_ARG_3)))) BIF_ERROR(BIF_P, BADARG); nrbytes = binary_size(BIF_ARG_1); ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); @@ -406,11 +422,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; + // XXX: Is there any reason to not just BIF_ERROR, so that the runtime + // survives? erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); hipe_flush_icache_range(address, nrbytes); + stp->text_segment = address; + stp->text_segment_size = nrbytes; hp = HAlloc(BIF_P, 3); hp[0] = make_arityval(2); hp[1] = address_to_term(address, BIF_P); @@ -423,25 +443,31 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) /* * Allocate memory for arbitrary non-Erlang data. */ -BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3) { - Uint align, nrbytes; - void *block; + Uint align; + HipeLoaderState *stp; + void *aligned_block; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || + (!(stp = get_loader_state(BIF_ARG_3))) || (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); - nrbytes = unsigned_val(BIF_ARG_2); - if (nrbytes == 0) + + if (stp->data_segment_size || stp->data_segment) + BIF_ERROR(BIF_P, BADARG); + + stp->data_segment_size = unsigned_val(BIF_ARG_2); + if (stp->data_segment_size == 0) BIF_RET(make_small(0)); - block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) { - fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", - __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); - erts_free(ERTS_ALC_T_HIPE, block); - BIF_ERROR(BIF_P, EXC_NOTSUP); - } - BIF_RET(address_to_term(block, BIF_P)); + + stp->data_segment_size += align-1; /* Make room to align the pointer */ + stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE_LL, stp->data_segment_size); + + /* Align the pointer */ + aligned_block = (void*)((UWord)(stp->data_segment + align - 1) + & ~(UWord)(align-1)); + BIF_RET(address_to_term(aligned_block, BIF_P)); } /* @@ -516,7 +542,7 @@ static void init_const_term_table(void) f.meta_alloc = (HMALLOC_FUN) erts_alloc; f.meta_free = (HMFREE_FUN) erts_free; f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &const_term_table, "const_term_table", 97, f); + hash_init(ERTS_ALC_T_HIPE_LL, &const_term_table, "const_term_table", 97, f); } BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) @@ -537,13 +563,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) BIF_RET(val); } -struct mfa_t { +struct hipe_mfa { Eterm mod; Eterm fun; Uint ari; }; -static int term_to_mfa(Eterm term, struct mfa_t *mfa) +static int term_to_mfa(Eterm term, struct hipe_mfa *mfa) { Eterm mod, fun, a; Uint ari; @@ -577,10 +603,7 @@ static void print_mfa(Eterm mod, Eterm fun, unsigned int ari) } #endif -/* - * Convert {M,F,A} to pointer to first insn after initial func_info. - */ -static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) +static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) { Module *modp; BeamCodeHeader* code_hdr; @@ -591,17 +614,17 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) return NULL; n = code_hdr->num_functions; for (i = 0; i < n; ++i) { - Uint *code_ptr = (Uint*)code_hdr->functions[i]; - ASSERT(code_ptr[0] == BeamOpCode(op_i_func_info_IaaI)); - if (code_ptr[3] == name && code_ptr[4] == arity) - return code_ptr+5; + ErtsCodeInfo *ci = code_hdr->functions[i]; + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + if (ci->mfa.function == name && ci->mfa.arity == arity) + return ci; } return NULL; } -Uint *hipe_bifs_find_pc_from_mfa(Eterm term) +ErtsCodeInfo* hipe_bifs_find_pc_from_mfa(Eterm term) { - struct mfa_t mfa; + struct hipe_mfa mfa; if (!term_to_mfa(term, &mfa)) return NULL; @@ -610,18 +633,26 @@ Uint *hipe_bifs_find_pc_from_mfa(Eterm term) BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) { - Eterm *pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ErtsCodeInfo* ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - BIF_RET(address_to_term(pc, BIF_P)); + BIF_RET(address_to_term(erts_codeinfo_to_code(ci), BIF_P)); +} + +BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) +{ + if (!erts_commit_hipe_patch_load(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(am_ok); } BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { - Eterm *pc; + ErtsCodeInfo *ci; void *address; int is_closure; - struct mfa_t mfa; + struct hipe_mfa mfa; switch (BIF_ARG_3) { case am_false: @@ -641,31 +672,26 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) simply have called hipe_bifs_find_pc_from_mfa(). */ if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); + ci = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); - if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif - hipe_set_call_trap(pc, address, is_closure); + if (ci) { + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", + erts_codeinfo_to_code(ci), address); + hipe_set_call_trap(ci, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; + HipeLoaderState* stp; + + stp = get_loader_state(BIF_ARG_2); + if (!stp) + BIF_ERROR(BIF_P, BADARG); sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { @@ -676,6 +702,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; + BIF_RET(NIL); } @@ -691,7 +724,7 @@ struct nbif { }; static struct nbif nbifs[BIF_SIZE] = { -#define BIF_LIST(MOD,FUN,ARY,CFUN,IX) \ +#define BIF_LIST(MOD,FUN,ARY,BIF,CFUN,IX) \ { {0,0}, MOD, FUN, ARY, &nbif_##CFUN }, #include "erl_bif_list.h" #undef BIF_LIST @@ -778,9 +811,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3) struct primop { HashBucket bucket; /* bucket.hvalue == atom_val(name) */ const void *address; -#if defined(__arm__) - void *trampoline; -#endif }; static struct primop primops[] = { @@ -824,7 +854,7 @@ static void init_primop_table(void) f.meta_free = (HMFREE_FUN) erts_free; f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &primop_table, "primop_table", 50, f); + hash_init(ERTS_ALC_T_HIPE_LL, &primop_table, "primop_table", 50, f); for (i = 0; i < sizeof(primops)/sizeof(primops[0]); ++i) hash_put(&primop_table, &primops[i]); @@ -839,29 +869,6 @@ static struct primop *primop_table_get(Eterm name) return hash_get(&primop_table, &tmpl); } -#if defined(__arm__) -static struct primop *primop_table_put(Eterm name) -{ - struct primop tmpl; - - init_primop_table(); - tmpl.bucket.hvalue = atom_val(name); - return hash_put(&primop_table, &tmpl); -} - -void *hipe_primop_get_trampoline(Eterm name) -{ - struct primop *primop = primop_table_get(name); - return primop ? primop->trampoline : NULL; -} - -void hipe_primop_set_trampoline(Eterm name, void *trampoline) -{ - struct primop *primop = primop_table_put(name); - primop->trampoline = trampoline; -} -#endif - /* * hipe_bifs_primop_address(Atom) -> address or false */ @@ -890,7 +897,8 @@ BIF_RETTYPE hipe_bifs_term_to_word_1(BIF_ALIST_1) } /* XXX: this is really a primop, not a BIF */ -BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1) +/* Called via standard_bif_interface_1 */ +BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1) { Eterm res; Eterm *hp; @@ -970,7 +978,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -992,35 +1000,40 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); fe->native_address = native_address; - if (erts_refc_dectest(&fe->refc, 0) == 0) + if (erts_smp_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { + HashBucket mod2mfa; struct { unsigned long hvalue; struct hipe_mfa_info *next; } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct ref *referred_from; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - void *trampoline; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; +#ifdef DEBUG + Export* dbg_export; #endif + }; static struct { @@ -1038,6 +1051,82 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; +Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ + +static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa) +{ + return mfa->mod2mfa.hvalue; +} + +static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa) +{ + return tmpl->hvalue != mfa->mod2mfa.hvalue; +} + +static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl) +{ + return tmpl; /* hash_put always use mfa itself at template */ +} + +static void mod2mfa_free(struct hipe_mfa_info* mfa) +{ +} + +static void mod2mfa_tab_init(void) +{ + HashFunctions f; + static int init_done = 0; + + if (init_done) + return; + init_done = 1; + + f.hash = (H_FUN) mod2mfa_hash; + f.cmp = (HCMP_FUN) mod2mfa_cmp; + f.alloc = (HALLOC_FUN) mod2mfa_alloc; + f.free = (HFREE_FUN) mod2mfa_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_HIPE_LL, &mod2mfa_tab, "mod2mfa_tab", 50, f); +} + +static struct hipe_mfa_info* mod2mfa_get(Module* modp) +{ + HashBucket tmpl; + tmpl.hvalue = modp->module; + return hash_get(&mod2mfa_tab, &tmpl); +} + +static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) +{ + mfa->mod2mfa.hvalue = atom_val(mfa->m); + return hash_put(&mod2mfa_tab, mfa); +} + + + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void *trampoline; +#endif + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1063,12 +1152,22 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } +static ERTS_INLINE +struct hipe_mfa_info* mod2mfa_get_safe(Module* modp) +{ + struct hipe_mfa_info* mfa; + hipe_mfa_info_table_rlock(); + mfa = mod2mfa_get(modp); + hipe_mfa_info_table_runlock(); + return mfa; +} + #define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) { unsigned long nbytes = size * sizeof(struct hipe_mfa_info*); - struct hipe_mfa_info **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + struct hipe_mfa_info **bucket = erts_alloc(ERTS_ALC_T_HIPE_LL, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -1097,25 +1196,25 @@ static void hipe_mfa_info_table_grow(void) b = next; } } - erts_free(ERTS_ALC_T_HIPE, old_bucket); + erts_free(ERTS_ALC_T_HIPE_LL, old_bucket); } static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigned int arity) { struct hipe_mfa_info *res; - res = (struct hipe_mfa_info*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*res)); + res = (struct hipe_mfa_info*)erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(*res)); res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - res->trampoline = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; +#ifdef DEBUG + res->dbg_export = NULL; #endif return res; @@ -1133,6 +1232,8 @@ void hipe_mfa_info_table_init(void) hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size); hipe_mfa_info_table_init_lock(); + + mod2mfa_tab_init(); } static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity) @@ -1154,21 +1255,12 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; + struct hipe_mfa_info *first_in_mod; unsigned int size; h = HIPE_MFA_HASH(m, f, arity); @@ -1189,216 +1281,137 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); - return p; -} -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) -{ - struct hipe_mfa_info *p; + first_in_mod = mod2mfa_put(p); + if (p != first_in_mod) { + p->next_in_mod = first_in_mod->next_in_mod; + first_in_mod->next_in_mod = p; + } + else { + p->next_in_mod = NULL; + } - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; - hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + + return p; } -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity) +static void remove_mfa_info(struct hipe_mfa_info* rm) { + unsigned int i; struct hipe_mfa_info *p; - void *trampoline; - - hipe_mfa_info_table_rlock(); - p = hipe_mfa_info_table_get_locked(m, f, arity); - trampoline = p ? p->trampoline : NULL; - hipe_mfa_info_table_runlock(); - return trampoline; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } } -void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline) +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); - p->trampoline = trampoline; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } -#endif BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct mfa_t mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; -} + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) -{ - struct hipe_mfa_info *p; - void *address; - - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; - - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,11 +1421,12 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ -BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) +/* Called via standard_bif_interface_3 */ +BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3) { Uint arity; void *address; @@ -1420,30 +1434,25 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } /* primop, but called like a BIF for error handling purposes */ -BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2) { Eterm hdr, m, f; void *address; @@ -1453,14 +1462,14 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) hdr = *boxed_val(BIF_ARG_1); if (is_export_header(hdr)) { Export *ep = (Export*)(export_val(BIF_ARG_1)[1]); - unsigned int actual_arity = ep->code[2]; + unsigned int actual_arity = ep->info.mfa.arity; if (actual_arity != BIF_ARG_2) goto badfun; - m = ep->code[0]; - f = ep->code[1]; + m = ep->info.mfa.module; + f = ep->info.mfa.function; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,73 +1480,31 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; -} + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) + return 0; -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; +} -struct ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { - struct mfa_t callee; + struct hipe_mfa callee; Eterm *tuple; - struct mfa_t caller; + struct hipe_mfa caller; void *address; void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; - struct ref *ref; + struct hipe_ref *ref; + HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1568,192 +1535,302 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (!trampoline) goto badarg; } - switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; - } + stp = get_loader_state(tuple[5]); + if (!stp) + goto badarg; + hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(struct hipe_ref)); ref->address = address; +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; +#endif ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)", + ref, callee.mod, callee.fun, callee.ari, ref->address); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)", + ref, caller.mod, caller.fun, caller.ari, ref->address); + BIF_RET(am_ok); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct mfa_t mfa; - const struct hipe_mfa_info *p; - struct ref *ref; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + p = hash_get(&mod2mfa_tab, unlink_me); + ASSERT(p); + if (p == unlink_me) { + hash_erase(&mod2mfa_tab, p); + if (p->next_in_mod) + mod2mfa_put(p->next_in_mod); + } + else { + struct hipe_mfa_info** prevp; + + do { + prevp = &p->next_in_mod; + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + } while (p != unlink_me); + + *prevp = p->next_in_mod; + } } -/* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. - * Test: Do init:restart in debug compiled vm with hipe compiled kernel. - */ -static void hipe_purge_all_refs(void) +static void purge_mfa(struct hipe_mfa_info* p) { - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE_LL, p); +} - hipe_mfa_info_table_rwlock(); +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + if (modp->old.hipe_code) { + if (modp->old.hipe_code->first_hipe_ref || + modp->old.hipe_code->first_hipe_sdesc) + return 1; + } + if (!modp->curr.code_hdr) { + return mod2mfa_get_safe(modp) != NULL; + } + return 0; +} - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - for (i = 0; i < nrbuckets; ++i) { - while (bucket[i] != NULL) { - struct hipe_mfa_info* mfa = bucket[i]; - bucket[i] = mfa->bucket.next; - - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } - erts_free(ERTS_ALC_T_HIPE, mfa); - } +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, + int is_blocking) +{ + struct hipe_ref* ref = first_ref; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + while (ref) { + struct hipe_ref* free_ref = ref; + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == caller_module); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + if (!is_blocking) + hipe_mfa_info_table_rwlock(); + unlink_mfa_from_mod(p); + purge_mfa(p); + if (!is_blocking) + hipe_mfa_info_table_rwunlock(); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE_LL, free_ref); } - hipe_mfa_info_table_rwunlock(); } -BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, + int is_blocking) { - struct mfa_t mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct ref **prev, *ref; + struct hipe_sdesc* sdesc = first_sdesc; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); - if (BIF_ARG_1 == am_all) { - hipe_purge_all_refs(); - BIF_RET(am_ok); + ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(make_atom(sdesc->m_aix) == module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); } +} - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + +void hipe_purge_module(Module* modp, int is_blocking) +{ + ASSERT(modp); + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + if (modp->old.hipe_code) { + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + if (modp->old.hipe_code->first_hipe_ref) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_ref = NULL; + } + + /* + * Remove all hipe_sdesc's for the old module instance + */ + if (modp->old.hipe_code->first_hipe_sdesc) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_sdesc = NULL; + } + + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; + } + + + /* + * Remove unreferred hipe_mfa_info's + * when all module instances are removed (like in init:restart) + */ + if (is_blocking && modp->curr.code_hdr == NULL) { + struct hipe_mfa_info* was_first = mod2mfa_get(modp); + struct hipe_mfa_info* is_first = was_first; + struct hipe_mfa_info** prevp = &is_first; + struct hipe_mfa_info *p; + + if (was_first) { + for (p = was_first ; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + if (p != was_first) + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + if (was_first != is_first) { + hash_erase(&mod2mfa_tab, was_first); + purge_mfa(was_first); + if (is_first) + mod2mfa_put(is_first); + } + } } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct mfa_t mfa; struct hipe_mfa_info *p; - struct ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else { +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void* trampoline = ref->trampoline; +#else + void* trampoline = NULL; +#endif + res = hipe_patch_call(ref->address, p->remote_address, trampoline); + } + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) @@ -1807,89 +1884,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value) } } -struct modinfo { - HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */ - unsigned int code_size; -}; - -static Hash modinfo_table; - -static HashValue modinfo_hash(void *tmpl) -{ - Eterm mod = (Eterm)tmpl; - return atom_val(mod); -} - -static int modinfo_cmp(void *tmpl, void *bucket) -{ - /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */ - return 0; -} - -static void *modinfo_alloc(void *tmpl) -{ - struct modinfo *p; - - p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p)); - p->code_size = 0; - return &p->bucket; -} - -static void init_modinfo_table(void) -{ - HashFunctions f; - static int init_done = 0; - - if (init_done) - return; - init_done = 1; - f.hash = (H_FUN) modinfo_hash; - f.cmp = (HCMP_FUN) modinfo_cmp; - f.alloc = (HALLOC_FUN) modinfo_alloc; - f.free = (HFREE_FUN) NULL; - f.meta_alloc = (HMALLOC_FUN) erts_alloc; - f.meta_free = (HMFREE_FUN) erts_free; - f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f); -} - -BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3) -{ - struct modinfo *p; - Sint code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1) || - is_not_small(BIF_ARG_3) || - (code_size = signed_val(BIF_ARG_3)) < 0) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1); - - if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */ - p->code_size += code_size; - else /* whole module */ - p->code_size = code_size; - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1) -{ - struct modinfo *p; - unsigned int code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1); - - code_size = p ? p->code_size : 0; - BIF_RET(make_small(code_size)); -} - BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3) { Uint *address, value; @@ -1925,3 +1919,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } + +BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) +{ + Binary *magic; + Eterm *hp; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + magic = hipe_alloc_loader_state(BIF_ARG_1); + + if (!magic) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + res = erts_mk_magic_ref(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->intern.refc, 1); + BIF_RET(res); +} diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..ee97e6fc64 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -2,7 +2,7 @@ * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,23 +26,20 @@ #ifndef HIPE_BIF0_H #define HIPE_BIF0_H -extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa); +extern ErtsCodeInfo *hipe_bifs_find_pc_from_mfa(Eterm mfa); extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); -extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); +extern BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); -extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline); -#endif -#if defined(__arm__) -extern void *hipe_primop_get_trampoline(Eterm name); -extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); -#endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking); +void hipe_purge_module(Module*, int is_blocking); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..4038ca7ef8 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2001-2016. All Rights Reserved. +# Copyright Ericsson AB 2001-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -44,22 +44,20 @@ bif hipe_bifs:ref/1 bif hipe_bifs:ref_get/1 bif hipe_bifs:ref_set/2 -bif hipe_bifs:enter_code/2 -bif hipe_bifs:alloc_data/2 +bif hipe_bifs:enter_code/3 +bif hipe_bifs:alloc_data/3 bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 bif hipe_bifs:fun_to_address/1 +bif hipe_bifs:commit_patch_load/1 bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:update_code_size/3 -bif hipe_bifs:code_size/1 - -bif hipe_bifs:enter_sdesc/1 +bif hipe_bifs:enter_sdesc/2 bif hipe_bifs:bif_address/3 bif hipe_bifs:primop_address/1 @@ -72,7 +70,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +82,8 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 -bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 + +bif hipe_bifs:alloc_loader_state/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 08adbd474e..3d3df4fd48 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -2,7 +2,7 @@ * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,73 +39,81 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) { + ErtsCodeInfo *ci; Eterm *pc; struct hipe_call_count *hcc; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] == BeamOpCode(op_hipe_trap_call)) BIF_ERROR(BIF_P, BADARG); if (pc[0] == BeamOpCode(op_hipe_call_count)) BIF_RET(NIL); - hcc = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hcc)); + hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc)); hcc->count = 0; hcc->opcode = pc[0]; - pc[-4] = (Eterm)hcc; + ci->u.hcc = hcc; pc[0] = BeamOpCode(op_hipe_call_count); BIF_RET(am_true); } BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1) { + ErtsCodeInfo* ci; Eterm *pc; struct hipe_call_count *hcc; unsigned count; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); - hcc = (struct hipe_call_count*)pc[-4]; + hcc = ci->u.hcc; count = hcc->count; pc[0] = hcc->opcode; - pc[-4] = (Eterm)NULL; - erts_free(ERTS_ALC_T_HIPE, hcc); + ci->u.hcc = NULL; + erts_free(ERTS_ALC_T_HIPE_SL, hcc); BIF_RET(make_small(count)); } BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1) { + ErtsCodeInfo* ci; Eterm *pc; struct hipe_call_count *hcc; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); - hcc = (struct hipe_call_count*)pc[-4]; + hcc = ci->u.hcc; BIF_RET(make_small(hcc->count)); } BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1) { + ErtsCodeInfo* ci; Eterm *pc; struct hipe_call_count *hcc; unsigned count; - pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); - if (!pc) + ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); + if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + pc = erts_codeinfo_to_code(ci); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); - hcc = (struct hipe_call_count*)pc[-4]; + hcc = ci->u.hcc; count = hcc->count; hcc->count = 0; BIF_RET(make_small(count)); @@ -124,757 +132,3 @@ BIF_RETTYPE hipe_bifs_trap_count_clear_0(BIF_ALIST_0) hipe_trap_count = 0; BIF_RET(make_small(count)); } - -/***************************************************************************** - * BIFs for benchmarking. These only do useful things if - * __BENCHMARK__ is defined in beam/benchmark.h. For documentation - * about how to add new counters or maintain the existing counters, - * see benchmark.h. - * - * If benchmarking is not enabled all BIFs will return false. If the - * required benchmark feature is not enabled, the counter will remain - * zero. - * - * process_info/0 -> { Number of live processes, - * Processes spawned in total } - * - * Live processes are increased when a new process is created, and - * decreased when a process dies. Processes spawned is increased - * when a process is created. - * - * - * process_info_clear/0 -> true - * - * Will reset the processes spawned-counters to zero. If this is - * done at some improper time, live processes may become a negative - * value. This is not a problem in itself, just as long as you know - * about it. - * - * - * message_info/0 -> { Messages sent, - * Messages copied, - * Ego messages (sender = receiver), - * Words sent, - * Words copied, - * Words preallocated } - * - * Counting the words sent in a shared heap system will affect - * runtime performance since it means that we have to calculate the - * size of the mesage. With private heaps, this is done anyway and - * will not affect performance. - * - * - * message_info_clear/0 -> true - * - * Reset the message counters to zero. - * - * - * message_sizes/0 -> true - * - * Displays a text-mode bar diagram with message sizes. There are no - * guaranties that this is printed in a way the Erlang system is - * supposed to print things. - * - * - * gc_info/0 -> { Minor collections, - * Major collections, - * Used heap, - * Allocated heap, - * Max used heap, - * Max allocated heap } - * - * Information about private heap garbage collections. Number of - * minor and major collections, how much heap is used and allocated - * and how much heap has been in use and allocated at most since the - * counters were reset. - * - * - * shared_gc_info/0 -> { Minor collections of the shared heap, - * Major collections of the shared heap, - * Used shared heap, - * Allocated shared heap, - * Max used shared heap, - * Max allocated shared heap } - * - * The same as above, but for the shared heap / message area. Note, - * that in a shared heap system the max used heap and max allocated - * heap are mostly the same, since the heap allways is filled before - * a garbage collection, and most garbage collections do not enlarge - * the heap. The private heap numbers are much more interesting. - * - * - * incremental_gc_info/0 -> { Complete minor GC cycles, - * Complete major GC cycles, - * Minor GC stages, - * Major GC stages } - * - * - * gc_info_clear/0 -> true - * - * Reset counters for both private and shared garbage collection. - * - * - * BM Timers - * --------- - * - * All timers returns tuples of the kind: { Minutes, Seconds, Milliseconds } - * except for the max times in garbage collection where times are normally - * small. The tuple is therefor: { Seconds, Milliseconds, Microseconds } - * - * system_timer/0 -> Mutator time - * - * This timer is not a real-time clock, it only runs when a process - * is scheduled to run. You can not find out the accual time a - * program has taken to run using this timer. - * - * - * system_timer_clear/0 -> true - * - * Reset system timer to zero. - * - * - * send_timer/0 -> { Send time, - * Copy time, - * Size time } - * - * Time spent in sending messages. The copy time and size time are - * only active if the copying is needed in send. Copying of data - * into ETS-tables etc is not timed with this timer. - * - * - * send_timer_clear/0 -> true - * - * Reset send timers to zero. - * - * - * gc_timer/0 -> { Time in minor collection, - * Time in major collection, - * Max time in minor collection (�s), - * Max time in major collection (�s) } - * - * Total time spent in garbage collection of the private heaps. The - * max times are for one separate collection. - * - * - * shared_gc_timer/0 -> { Time in minor collection, - * Time in major collection, - * Max time in minor collection (�s), - * Max time in major collection (�s) } - * - * Total time spent in garbage collection of the shared heap / - * message area. The max times are for one separate collection. - * - * - * gc_timer_clear/0 -> true - * - * Reset private and shared garbage collection timers to zero. Note, - * that the max-times are also reset. - * - * - * misc_timer/0 -> { Misc 0, Misc 1, Misc 2 } - * - * Timers for debug purposes. In a normal system, these timers are - * never used. Add these timers at places where you want to time - * something not covered here. Use BM_SWAP_TIMER(from,to) to start - * one of the misc timers. - * - * ... code timed by the system timer ... - * BM_SWAP_TIMER(system,misc1); - * ... code we want to time ... - * BM_SWAP_TIMER(misc1,system); - * ... back on system time ... - * - * - * misc_timer_clear/0 -> true - * - * Reset misc timers to zero. - */ - -BIF_RETTYPE hipe_bifs_process_info_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ -#ifndef BM_COUNTERS - Uint processes_busy = 0; - Uint processes_spawned = 0; -#endif - Eterm *hp; - - hp = HAlloc(BIF_P, 3); - BIF_RET(TUPLE2(hp, - make_small(processes_busy), - make_small(processes_spawned))); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_process_info_clear_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ -#ifdef BM_COUNTERS - processes_spawned = 0; -#endif - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_message_info_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - Eterm *hp; -#ifndef BM_COUNTERS - unsigned long messages_sent = 0; - unsigned long messages_copied = 0; - unsigned long messages_ego = 0; -#endif -#ifndef BM_MESSAGE_SIZES - unsigned long words_sent = 0; - unsigned long words_copied = 0; - unsigned long words_prealloc = 0; -#endif - - hp = HAlloc(BIF_P, 7); - BIF_RET(TUPLE6(hp, - make_small(messages_sent), - make_small(messages_copied), - make_small(messages_ego), - make_small(words_sent), - make_small(words_copied), - make_small(words_prealloc))); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_message_info_clear_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ -#ifdef BM_COUNTERS - messages_sent = 0; - messages_copied = 0; - messages_ego = 0; -#endif -#ifdef BM_MESSAGE_SIZES - words_sent = 0; - words_copied = 0; - words_prealloc = 0; - { - int i; - for (i = 0; i < 1000; i++) - message_sizes[i] = 0; - } -#endif - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_message_sizes_0(BIF_ALIST_0) -{ -#ifdef BM_MESSAGE_SIZES - int i, j, max = 0; - int tmp[12] = {0,0,0,0,0,0,0,0,0,0,0,0}; - - for (i = 0; i < 65; i++) { - tmp[0] += message_sizes[i]; - if (tmp[0] > max) - max = tmp[0]; - } - for (i = 65; i < 999; i++) { - tmp[i / 100 + 1] += message_sizes[i]; - if (tmp[i / 100 + 1] > max) - max = tmp[i / 100 + 1]; - } - tmp[11] = message_sizes[999]; - if (tmp[11] > max) - max = tmp[11]; - for (i = -1; i < 11; i++) { - int num = (tmp[i + 1] * 50) / max; - if (i == -1) - printf("\n\r 0 - 64: (%6d) |", tmp[0]); - else if (i == 0) - printf("\n\r 65 - 99: (%6d) |", tmp[1]); - else if (i == 10) - printf("\n\r >= 1000: (%6d) |", tmp[11]); - else - printf("\n\r%3d - %3d: (%6d) |", i * 100, i * 100 + 99, - tmp[i + 1]); - - for (j = 0; j < num; j++) - printf("."); - } - printf("\n\r"); - - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_gc_info_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ -#ifndef BM_COUNTERS - Uint minor_gc = 0; - Uint major_gc = 0; -#endif -#ifndef BM_HEAP_SIZES - Uint max_used_heap = 0; - Uint max_allocated_heap = 0; -#endif - Eterm *hp; - Uint used_heap = (BIF_P->htop - BIF_P->heap) + - (OLD_HTOP(BIF_P) - OLD_HEAP(BIF_P)) + - MBUF_SIZE(BIF_P); - - Uint alloc_heap = (BIF_P->hend - BIF_P->heap) + - (OLD_HEND(BIF_P) - OLD_HEAP(BIF_P)) + - MBUF_SIZE(BIF_P); - - hp = HAlloc(BIF_P, 7); - BIF_RET(TUPLE6(hp, - make_small((Uint)minor_gc), - make_small((Uint)major_gc), - make_small((Uint)used_heap), - make_small((Uint)alloc_heap), - make_small(max_used_heap), - make_small(max_allocated_heap))); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_shared_gc_info_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ -#if !(defined(BM_COUNTERS)) - Uint minor_global_gc = 0; - Uint major_global_gc = 0; -#endif -#ifndef BM_HEAP_SIZES - Uint max_used_global_heap = 0; - Uint max_allocated_global_heap = 0; -#endif - Eterm *hp; - - Uint tmp_used_heap = 0; - Uint tmp_allocated_heap = 0; - - hp = HAlloc(BIF_P, 7); - BIF_RET(TUPLE6(hp, - make_small((uint)minor_global_gc), - make_small((uint)major_global_gc), - make_small(tmp_used_heap), - make_small(tmp_allocated_heap), - make_small(max_used_global_heap), - make_small(max_allocated_global_heap))); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_incremental_gc_info_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ -#if !defined(BM_COUNTERS) - Uint minor_gc_cycles = 0; - Uint major_gc_cycles = 0; - Uint minor_gc_stages = 0; - Uint major_gc_stages = 0; -#endif - Eterm *hp; - - hp = HAlloc(BIF_P, 5); - BIF_RET(TUPLE4(hp, - make_small(minor_gc_cycles), - make_small(major_gc_cycles), - make_small(minor_gc_stages), - make_small(major_gc_stages))); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_gc_info_clear_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - -#ifdef BM_COUNTERS - minor_gc = 0; - major_gc = 0; -#endif - -#ifdef BM_HEAP_SIZES - max_used_heap = 0; - max_allocated_heap = 0; - max_used_global_heap = 0; - max_allocated_global_heap = 0; -#endif - - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_pause_times_0(BIF_ALIST_0) -{ -#ifdef BM_TIMERS - int i; - int total_time = 0, n = 0; - int left = 0, right = 0, mid = 0; - - printf("Pause times in minor collection:\r\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (pause_times[i] > 0) { - printf("%d: %ld\r\n", i, pause_times[i]); - total_time += pause_times[i] * i; - n += pause_times[i]; - - if (i > mid) - right += pause_times[i]; - - while (right > left) { - left += pause_times[mid++]; - right -= pause_times[mid]; - } - } - } - - printf("Number of collections: %d\r\n", n); - printf("Total collection time: %d\r\n", total_time); - if (n > 0) - printf("Mean pause time: %d\r\n", total_time / n); - - printf("Geometrical mean: %d\r\n", mid); - - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - printf("Pause times in major collection:\r\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (pause_times_old[i] > 0) { - printf("%d: %ld\r\n", i, pause_times_old[i]); - total_time += pause_times_old[i] * i; - n += pause_times_old[i]; - } - } - - printf("Number of collections: %d\r\n", n); - printf("Total collection time: %d\r\n", total_time); - if (n > 0) - printf("Mean pause time: %d\r\n", total_time / n); - - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -/* XXX: these macros have free variables */ -#ifdef BM_TIMERS -#define MAKE_TIME(_timer_) { \ - BM_TIMER_T tmp = _timer_##_time / 1000000; \ - milli = tmp % 1000; \ - tmp /= 1000; \ - sec = tmp % 60; \ - min = tmp / 60; } - -#define MAKE_MICRO_TIME(_timer_) { \ - BM_TIMER_T tmp = _timer_##_time / 1000; \ - micro = tmp % 1000; \ - tmp /= 1000; \ - milli = tmp % 1000; \ - sec = tmp / 1000; } - -#else -#define MAKE_TIME(_timer_) -#define MAKE_MICRO_TIME(_timer_) -#endif - -BIF_RETTYPE hipe_bifs_system_timer_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - uint min = 0; - uint sec = 0; - uint milli = 0; - Eterm *hp; - - hp = HAlloc(BIF_P, 4); - MAKE_TIME(system); - BIF_RET(TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli))); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_system_timer_clear_0(BIF_ALIST_0) -{ -#ifdef BM_TIMERS - system_time = 0; - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_send_timer_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - uint min = 0; - uint sec = 0; - uint milli = 0; - Eterm *hp; - Eterm sendtime, copytime, sizetime; - - hp = HAlloc(BIF_P, 4 * 4); - - MAKE_TIME(send); - sendtime = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_TIME(copy); - copytime = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_TIME(size); - sizetime = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - BIF_RET(TUPLE3(hp, sendtime, copytime, sizetime)); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_send_timer_clear_0(BIF_ALIST_0) -{ -#ifdef BM_TIMERS - send_time = 0; - copy_time = 0; - size_time = 0; - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_gc_timer_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - Eterm *hp; - uint min = 0; - uint sec = 0; - uint milli = 0; - uint micro = 0; - Eterm minor, major, max_min, max_maj; - - hp = HAlloc(BIF_P, 4 * 4 + 5); - - MAKE_TIME(minor_gc); - minor = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_TIME(major_gc); - major = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_MICRO_TIME(max_minor); - max_min = TUPLE3(hp, - make_small(sec), - make_small(milli), - make_small(micro)); - hp += 4; - - MAKE_MICRO_TIME(max_major); - max_maj = TUPLE3(hp, - make_small(sec), - make_small(milli), - make_small(micro)); - hp += 4; - - BIF_RET(TUPLE4(hp, minor, major, max_min, max_maj)); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_shared_gc_timer_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - Eterm *hp; - uint min = 0; - uint sec = 0; - uint milli = 0; - uint micro = 0; - Eterm minor, major, max_min, max_maj; - - hp = HAlloc(BIF_P, 4 * 4 + 5); - - MAKE_TIME(minor_global_gc); - minor = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_TIME(major_global_gc); - major = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_MICRO_TIME(max_global_minor); - max_min = TUPLE3(hp, - make_small(sec), - make_small(milli), - make_small(micro)); - hp += 4; - - MAKE_MICRO_TIME(max_global_major); - max_maj = TUPLE3(hp, - make_small(sec), - make_small(milli), - make_small(micro)); - hp += 4; - - BIF_RET(TUPLE4(hp, minor, major, max_min, max_maj)); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_gc_timer_clear_0(BIF_ALIST_0) -{ -#ifdef BM_TIMERS - minor_gc_time = 0; - major_gc_time = 0; - max_minor_time = 0; - max_major_time = 0; - minor_global_gc_time = 0; - major_global_gc_time = 0; - max_global_minor_time = 0; - max_global_major_time = 0; - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_misc_timer_0(BIF_ALIST_0) -{ -#ifdef __BENCHMARK__ - uint min = 0; - uint sec = 0; - uint milli = 0; - Eterm *hp; - Eterm misctime1, misctime2, misctime3; - - hp = HAlloc(BIF_P, 4 * 4); - - MAKE_TIME(misc0); - misctime1 = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_TIME(misc1); - misctime2 = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - - MAKE_TIME(misc2); - misctime3 = TUPLE3(hp, - make_small(min), - make_small(sec), - make_small(milli)); - hp += 4; - BIF_RET(TUPLE3(hp, misctime1, misctime2, misctime3)); -#else - BIF_RET(am_false); -#endif -} - -BIF_RETTYPE hipe_bifs_misc_timer_clear_0(BIF_ALIST_0) -{ -#ifdef BM_TIMERS - misc0_time = 0; - misc1_time = 0; - misc2_time = 0; - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif -} - -#undef MAKE_TIME -#undef MAKE_MICRO_TIME - -/* - * HiPE hrvtime(). - * These implementations are currently available: - * + The fallback, which is the same as {X,_} = runtime(statistics). - */ - -static double fallback_get_hrvtime(void) -{ - unsigned long ms_user; - - elapsed_time_both(&ms_user, NULL, NULL, NULL); - return (double)ms_user; -} - -/* - * Fallback, if nothing better exists. - * This is the same as {X,_} = statistics(runtime), which uses - * times(2) on Unix systems. - */ - -#define hrvtime_is_started() 1 -#define start_hrvtime() do{}while(0) -#define stop_hrvtime() do{}while(0) -#define get_hrvtime() fallback_get_hrvtime() - -BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0) -{ - Eterm *hp; - Eterm res; - FloatDef f; - - if (!hrvtime_is_started()) - start_hrvtime(); - f.fd = get_hrvtime(); - hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT); - res = make_float(hp); - PUT_DOUBLE(f, hp); - BIF_RET(res); -} - -BIF_RETTYPE hipe_bifs_stop_hrvtime_0(BIF_ALIST_0) -{ - stop_hrvtime(); - BIF_RET(am_true); -} diff --git a/erts/emulator/hipe/hipe_bif1.tab b/erts/emulator/hipe/hipe_bif1.tab index c5b452f199..4be0ad0e9c 100644 --- a/erts/emulator/hipe/hipe_bif1.tab +++ b/erts/emulator/hipe/hipe_bif1.tab @@ -27,24 +27,3 @@ bif hipe_bifs:call_count_get/1 bif hipe_bifs:call_count_clear/1 bif hipe_bifs:trap_count_get/0 bif hipe_bifs:trap_count_clear/0 -bif hipe_bifs:process_info/0 -bif hipe_bifs:process_info_clear/0 -bif hipe_bifs:message_info/0 -bif hipe_bifs:message_info_clear/0 -bif hipe_bifs:message_sizes/0 -bif hipe_bifs:gc_info/0 -bif hipe_bifs:shared_gc_info/0 -bif hipe_bifs:incremental_gc_info/0 -bif hipe_bifs:gc_info_clear/0 -bif hipe_bifs:pause_times/0 -bif hipe_bifs:system_timer/0 -bif hipe_bifs:system_timer_clear/0 -bif hipe_bifs:send_timer/0 -bif hipe_bifs:send_timer_clear/0 -bif hipe_bifs:gc_timer/0 -bif hipe_bifs:shared_gc_timer/0 -bif hipe_bifs:gc_timer_clear/0 -bif hipe_bifs:misc_timer/0 -bif hipe_bifs:misc_timer_clear/0 -bif hipe_bifs:get_hrvtime/0 -bif hipe_bifs:stop_hrvtime/0 diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index dfd34e31d4..e04d3d32d1 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -155,7 +155,7 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0) #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1); +BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1); # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ @@ -163,13 +163,13 @@ BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1); # define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) -BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1) +BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1) { - typedef BIF_RETTYPE Bif(BIF_ALIST_1); - Bif* fp = (Bif*) (BIF_P->hipe.bif_callee); + typedef BIF_RETTYPE nBif(NBIF_ALIST_1); + nBif* fp = (nBif*) (BIF_P->hipe.bif_callee); BIF_RETTYPE res; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P); - res = (*fp)(BIF_P, BIF__ARGS); + res = (*fp)(NBIF_CALL_ARGS); ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P); return res; } diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index 29095a5389..f034c4700c 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -71,6 +71,32 @@ ****************************************************************/ /* + * NOTE: + * Beam BIFs have the prototype: + * Eterm (*BIF)(Process *c_p, Eterm *regs, UWord *I) + * Native BIFs have the prototype: + * Eterm (*BIF)(Process *c_p, Eterm *regs) + * + * Beam BIFs expect 'I' to contain current instruction + * pointer when called from beam, and expect 'I' to + * contain a pointer to the export entry of the BIF + * when called from native code. In order to facilitate + * this, beam BIFs are called via wrapper functions + * when called from native code. These wrapper functions + * are auto-generated (by utils/make_tables) and have + * the function names nbif_impl_<BIF>. + * + * The standard_bif_interface_*() and + * gc_bif_interface_*() will add the prefix and + * thus call nbif_impl_<cbif_name>. That is, all + * functions (true BIFs as well as other c-functions) + * called via these interfaces have to be named + * nbif_impl_<FUNC>. + */ + +/* + * See NOTE above! + * * standard_bif_interface_0(nbif_name, cbif_name) * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) @@ -93,9 +119,12 @@ */ /* + * See NOTE above! + * * gc_bif_interface_0(nbif_name, cbif_name) * gc_bif_interface_1(nbif_name, cbif_name) * gc_bif_interface_2(nbif_name, cbif_name) + * gc_bif_interface_3(nbif_name, cbif_name) * * A BIF which may do a GC or walk the native stack. * May read NSP, NSP_LIMIT, NRA, HP, HP_LIMIT, and FCALLS. @@ -152,9 +181,9 @@ standard_bif_interface_0(nbif_ports_0, ports_0) * BIFs and primops that may do a GC (change heap limit and walk the native stack). * XXX: erase/1 and put/2 cannot fail */ -gc_bif_interface_2(nbif_erts_internal_check_process_code_2, hipe_erts_internal_check_process_code_2) +gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1) gc_bif_interface_1(nbif_erase_1, erase_1) -gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0) +gc_bif_interface_1(nbif_erts_internal_garbage_collect_1, erts_internal_garbage_collect_1) gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc) gc_bif_interface_2(nbif_put_2, put_2) @@ -246,7 +275,7 @@ nocons_nofail_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_intege noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer) ')dnl -gc_bif_interface_0(nbif_check_get_msg, hipe_check_get_msg) +nofail_primop_interface_0(nbif_check_get_msg, hipe_check_get_msg) #`ifdef' NO_FPE_SIGNALS nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe) @@ -263,33 +292,35 @@ noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc) ',)dnl /* - * Standard BIFs. - * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index) + * BIFs that disable GC while trapping are called via a wrapper + * to reserve stack space for the "trap frame". + * They occasionally need to call the garbage collector in order to make room + * for the trap frame on the BEAM stack. */ +gc_bif_interface_1(nbif_term_to_binary_1, hipe_wrapper_term_to_binary_1) +gc_bif_interface_2(nbif_term_to_binary_2, hipe_wrapper_term_to_binary_2) +gc_bif_interface_1(nbif_binary_to_term_1, hipe_wrapper_binary_to_term_1) +gc_bif_interface_2(nbif_binary_to_term_2, hipe_wrapper_binary_to_term_2) +gc_bif_interface_1(nbif_binary_to_list_1, hipe_wrapper_binary_to_list_1) +gc_bif_interface_3(nbif_binary_to_list_3, hipe_wrapper_binary_to_list_3) +gc_bif_interface_1(nbif_bitstring_to_list_1, hipe_wrapper_bitstring_to_list_1) +gc_bif_interface_1(nbif_list_to_binary_1, hipe_wrapper_list_to_binary_1) +gc_bif_interface_1(nbif_iolist_to_binary_1, hipe_wrapper_iolist_to_binary_1) +gc_bif_interface_1(nbif_binary_list_to_bin_1, hipe_wrapper_binary_list_to_bin_1) +gc_bif_interface_1(nbif_list_to_bitstring_1, hipe_wrapper_list_to_bitstring_1) +gc_bif_interface_2(nbif_send_2, hipe_wrapper_send_2) +gc_bif_interface_3(nbif_send_3, hipe_wrapper_send_3) +gc_bif_interface_2(nbif_ebif_bang_2, hipe_wrapper_ebif_bang_2) +gc_bif_interface_2(nbif_maps_merge_2, hipe_wrapper_maps_merge_2) -/* BIFs that disable GC while trapping are called via a wrapper - * to reserve stack space for the "trap frame". + +/* + * Standard BIFs. + * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index) */ -define(CFUN,`ifelse( -$1, term_to_binary_1, hipe_wrapper_$1, -$1, term_to_binary_2, hipe_wrapper_$1, -$1, binary_to_term_1, hipe_wrapper_$1, -$1, binary_to_term_2, hipe_wrapper_$1, -$1, binary_to_list_1, hipe_wrapper_$1, -$1, binary_to_list_3, hipe_wrapper_$1, -$1, bitstring_to_list_1, hipe_wrapper_$1, -$1, list_to_binary_1, hipe_wrapper_$1, -$1, iolist_to_binary_1, hipe_wrapper_$1, -$1, binary_list_to_bin_1, hipe_wrapper_$1, -$1, list_to_bitstring_1, hipe_wrapper_$1, -$1, send_2, hipe_wrapper_$1, -$1, send_3, hipe_wrapper_$1, -$1, ebif_bang_2, hipe_wrapper_$1, -$1, maps_merge_2, hipe_wrapper_$1, -$1)') -define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, CFUN($4))') -include(TARGET/`erl_bif_list.h') +define(BIF_LIST,`standard_bif_interface_$3(nbif_$5, $5)') +include(TTF_DIR/`erl_bif_list.h') /* * Guard BIFs. diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c index ace489452f..222a11db3d 100644 --- a/erts/emulator/hipe/hipe_debug.c +++ b/erts/emulator/hipe/hipe_debug.c @@ -62,10 +62,12 @@ static void print_beam_pc(BeamInstr *pc) } else if (pc == &beam_apply[1]) { printf("normal-process-exit"); } else { - BeamInstr *mfa = find_function_from_pc(pc); - if (mfa) + ErtsCodeMFA *cmfa = find_function_from_pc(pc); + if (cmfa) erts_printf("%T:%T/%bpu + 0x%bpx", - mfa[0], mfa[1], mfa[2], pc - &mfa[3]); + cmfa->module, cmfa->function, + cmfa->arity, + pc - erts_codemfa_to_code(cmfa)); else printf("?"); } @@ -214,10 +216,10 @@ void hipe_print_pcb(Process *p) U("seq..clock ", seq_trace_clock); U("seq..astcnt", seq_trace_lastcnt); U("seq..token ", seq_trace_token); - U("intial[0] ", u.initial[0]); - U("intial[1] ", u.initial[1]); - U("intial[2] ", u.initial[2]); - P("current ", current); + U("intial.mod ", u.initial.module); + U("intial.fun ", u.initial.function); + U("intial.ari ", u.initial.arity); + U("current ", current); P("cp ", cp); P("i ", i); U("catches ", catches); diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index d0619a0609..1a4a4c7952 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2016. All Rights Reserved. + * Copyright Ericsson AB 2004-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -46,6 +46,8 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; + ASSERT(!p->hipe.gc_is_unsafe); + if (!p->hipe.nstack) { ASSERT(!p->hipe.nsp && !p->hipe.nstend); return n_htop; @@ -89,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (!erts_is_literal(gval, ptr)) { - MOVE_BOXED(ptr, val, n_htop, nsp_i); + move_boxed(&ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -97,8 +99,8 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { - ASSERT(within(ptr, p)); - MOVE_CONS(ptr, val, n_htop, nsp_i); + ASSERT(erts_dbg_within_proc(ptr, p, NULL)); + move_cons(&ptr, val, &n_htop, nsp_i); } } } @@ -121,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -136,6 +138,8 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) char *mature; Uint mature_size; + ASSERT(!p->hipe.gc_is_unsafe); + if (!p->hipe.nstack) { ASSERT(!p->hipe.nsp && !p->hipe.nstend); return; @@ -202,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_BOXED(ptr, val, old_htop, nsp_i); + move_boxed(&ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - ASSERT(within(ptr, p)); - MOVE_BOXED(ptr, val, n_htop, nsp_i); + ASSERT(erts_dbg_within_proc(ptr, p, NULL)); + move_boxed(&ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -213,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - MOVE_CONS(ptr, val, old_htop, nsp_i); + move_cons(&ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - ASSERT(within(ptr, p)); - MOVE_CONS(ptr, val, n_htop, nsp_i); + ASSERT(erts_dbg_within_proc(ptr, p, NULL)); + move_cons(&ptr, val, &n_htop, nsp_i); } } } @@ -233,3 +237,155 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) } abort(); } + +Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, + Uint area_size) +{ + /* known nstack walk state */ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + ASSERT(!p->hipe.gc_is_unsafe); + + if (!p->hipe.nstack) { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + return old_htop; + } + if (!nstack_walk_init_check(p)) + return old_htop; + + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned long ra; + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned i = 0; + unsigned mask = sdesc->livebits[0]; + for (;;) { + if (mask & 1) { + Eterm *nsp_i = nstack_walk_frame_index(nsp, i); + Eterm gval = *nsp_i; + if (is_boxed(gval)) { + Eterm *ptr = boxed_val(gval); + Eterm val = *ptr; + if (IS_MOVED_BOXED(val)) { + ASSERT(is_boxed(val)); + *nsp_i = val; + } else if (ErtsInArea(ptr, area, area_size)) { + move_boxed(&ptr, val, &old_htop, nsp_i); + } + } else if (is_list(gval)) { + Eterm *ptr = list_val(gval); + Eterm val = *ptr; + if (IS_MOVED_CONS(val)) { + *nsp_i = ptr[1]; + } else if (ErtsInArea(ptr, area, area_size)) { + move_cons(&ptr, val, &old_htop, nsp_i); + } + } + } + if (++i >= sdesc_size) + break; + if (i & 31) + mask >>= 1; + else + mask = sdesc->livebits[i >> 5]; + } + ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return old_htop; +} + +int +nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + ASSERT(!rp->hipe.gc_is_unsafe); + + if (!rp->hipe.nstack || !nstack_walk_init_check(rp)) return 0; + ASSERT(rp->hipe.nsp && rp->hipe.nstend); + nsp = nstack_walk_nsp_begin(rp); + nsp_end = nstack_walk_nsp_end(rp); + sdesc = nstack_walk_init_sdesc_ignore_trap(rp, &walk_state); + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned long ra; + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned i = 0; + unsigned mask = sdesc->livebits[0]; + for (;;) { + if (mask & 1) { + Eterm *nsp_i = nstack_walk_frame_index(nsp, i); + Eterm val = *nsp_i; + switch (primary_tag(val)) { + case TAG_PRIMARY_BOXED: + case TAG_PRIMARY_LIST: + if (ErtsInArea(val, mod_start, mod_size)) { + return 1; + } + break; + } + } + if (++i >= sdesc_size) + break; + if (i & 31) + mask >>= 1; + else + mask = sdesc->livebits[i >> 5]; + } + ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)rp->hipe.ngra; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} + +int +nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + if (!p->hipe.nstack || !nstack_walk_init_check(p)) + return 0; + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + /* Check the topmost frame */ + if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size)) + return 1; + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned long ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + if (ErtsInArea(ra, seg_start, seg_size)) + return 1; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c new file mode 100644 index 0000000000..27d7bb9dee --- /dev/null +++ b/erts/emulator/hipe/hipe_load.c @@ -0,0 +1,106 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016-2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.c + * + * HiPE atomic code loader + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "global.h" +#include "erl_binary.h" +#include "hipe_load.h" +#include "hipe_bif0.h" + +void hipe_free_loader_state(HipeLoaderState *stp) +{ + if (stp->module == NIL) return; + + // TODO: Needs to be freed separately. We'd like have a unified executable + // code allocator, so postpone this for now. + /* if (stp->text_segment) */ + /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */ + stp->text_segment = NULL; + stp->text_segment_size = 0; + + if (stp->data_segment) + erts_free(ERTS_ALC_T_HIPE_LL, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; + + if (stp->new_hipe_refs) { + hipe_purge_refs(stp->new_hipe_refs, stp->module, 0); + stp->new_hipe_refs = NULL; + } + if (stp->new_hipe_sdesc) { + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0); + stp->new_hipe_sdesc = NULL; + } + + stp->module = NIL; +} + +static int +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor); + + hipe_free_loader_state(stp); + return 1; +} + +Binary *hipe_alloc_loader_state(Eterm module) +{ + HipeLoaderState *stp; + Binary *magic; + + if (is_not_atom(module)) return NULL; + + magic = erts_create_magic_binary(sizeof(HipeLoaderState), + hipe_loader_state_dtor); + erts_refc_inc(&magic->intern.refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); + + stp->module = module; + stp->text_segment = NULL; + stp->text_segment_size = 0; + stp->data_segment = NULL; + stp->data_segment_size = 0; + + stp->new_hipe_refs = NULL; + stp->new_hipe_sdesc = NULL; + + return magic; +} + +HipeLoaderState * +hipe_get_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return NULL; + + return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic); +} diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h new file mode 100644 index 0000000000..40c8a8aa2a --- /dev/null +++ b/erts/emulator/hipe/hipe_load.h @@ -0,0 +1,48 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.h + * + * HiPE atomic code loader + */ +#ifndef HIPE_LOAD_H +#define HIPE_LOAD_H + +#include "global.h" + +typedef struct hipe_loader_state { + Eterm module; /* Module name, atom */ + + void *text_segment; + Uint text_segment_size; + + void *data_segment; + Uint data_segment_size; + + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; + +} HipeLoaderState; + +extern Binary *hipe_alloc_loader_state(Eterm module); +extern void hipe_free_loader_state(HipeLoaderState*); +extern HipeLoaderState *hipe_get_loader_state(Binary *binary); + +#endif /* HIPE_LOAD_H */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 0d3493ec6c..4573980e1e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = { presence or absence of struct erl_fun_thing's "next" field. */ { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) }, { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) }, -#ifdef HIPE - { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) }, -#endif { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) }, { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) }, { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) }, @@ -525,6 +522,12 @@ static const struct rts_param rts_params[] = { { 51, "P_CALLEE_EXP", 1, offsetof(struct process, hipe.u.callee_exp) }, { 52, "THE_NON_VALUE", 1, (int)THE_NON_VALUE }, + + { 53, "P_GCUNSAFE", +#ifdef DEBUG + 1, offsetof(struct process, hipe.gc_is_unsafe) +#endif + }, }; #define NR_PARAMS ARRAY_SIZE(rts_params) diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index ed95045292..ba7ae1e6a8 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -2,7 +2,7 @@ * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -174,43 +174,17 @@ void hipe_mode_switch_init(void) make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL)); hipe_mfa_info_table_init(); - -#if (defined(__i386__) || defined(__x86_64__)) && defined(__linux__) - /* Verify that the offset of c-p->hipe does not change. - The ErLLVM hipe backend depends on it being in a specific - position. Kostis et al has promised to fix this in upstream - llvm by OTP 20, so it should be possible to remove these asserts - after that. */ - ERTS_CT_ASSERT(sizeof(ErtsPTabElementCommon) == - (sizeof(Eterm) + /* id */ - sizeof(((ErtsPTabElementCommon*)0)->refc) + - sizeof(ErtsTracer) + /* tracer */ - sizeof(Uint) + /* trace_flags */ - sizeof(erts_smp_atomic_t) + /* timer */ - sizeof(((ErtsPTabElementCommon*)0)->u))); - - ERTS_CT_ASSERT(offsetof(Process, hipe) == - (sizeof(ErtsPTabElementCommon) + /* common */ - sizeof(Eterm*) + /* htop */ - sizeof(Eterm*) + /* stop */ - sizeof(Eterm*) + /* heap */ - sizeof(Eterm*) + /* hend */ - sizeof(Uint) + /* heap_sz */ - sizeof(Uint) + /* min_heap_size */ - sizeof(Uint) + /* min_vheap_size */ - sizeof(volatile unsigned long))); /* fp_exception */ -#endif - } -void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure) +void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure) { - HIPE_ASSERT(bfun[-5] == BeamOpCode(op_i_func_info_IaaI)); + BeamInstr* bfun = erts_codeinfo_to_code(ci); + HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); bfun[0] = is_closure ? BeamOpCode(op_hipe_trap_call_closure) : BeamOpCode(op_hipe_trap_call); - bfun[-4] = (Uint)nfun; + ci->u.ncallee = (void (*)(void)) nfun; } static __inline__ void @@ -691,7 +665,7 @@ void hipe_inc_nstack(Process *p) { unsigned old_size = p->hipe.nstend - p->hipe.nstack; unsigned new_size = hipe_next_nstack_size(old_size); - Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE, new_size*sizeof(Eterm)); + Eterm *new_nstack = erts_alloc(ERTS_ALC_T_HIPE_STK, new_size*sizeof(Eterm)); unsigned used_size = p->hipe.nstend - p->hipe.nsp; sys_memcpy(new_nstack+new_size-used_size, p->hipe.nsp, used_size*sizeof(Eterm)); @@ -700,7 +674,7 @@ void hipe_inc_nstack(Process *p) if (p->hipe.nstblacklim) p->hipe.nstblacklim = new_nstack + new_size - (p->hipe.nstend - p->hipe.nstblacklim); if (p->hipe.nstack) - erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack); + erts_free(ERTS_ALC_T_HIPE_STK, p->hipe.nstack); p->hipe.nstack = new_nstack; p->hipe.nstend = new_nstack + new_size; p->hipe.nsp = new_nstack + new_size - used_size; @@ -710,7 +684,7 @@ void hipe_inc_nstack(Process *p) void hipe_empty_nstack(Process *p) { if (p->hipe.nstack) { - erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack); + erts_free(ERTS_ALC_T_HIPE_STK, p->hipe.nstack); } p->hipe.nstgraylim = NULL; p->hipe.nsp = NULL; @@ -718,12 +692,9 @@ void hipe_empty_nstack(Process *p) p->hipe.nstend = NULL; } -void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free) +void hipe_set_closure_stub(ErlFunEntry *fe) { - unsigned arity; - - arity = fe->arity; - fe->native_address = (Eterm*) hipe_closure_stub_address(arity); + fe->native_address = (Eterm*) hipe_closure_stub_address(fe->arity); } Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s) diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h index c40077d558..3c042913f6 100644 --- a/erts/emulator/hipe/hipe_mode_switch.h +++ b/erts/emulator/hipe/hipe_mode_switch.h @@ -2,7 +2,7 @@ * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,11 +55,11 @@ extern int hipe_modeswitch_debug; void hipe_mode_switch_init(void); -void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure); +void hipe_set_call_trap(ErtsCodeInfo*, void *nfun, int is_closure); Process *hipe_mode_switch(Process*, unsigned, Eterm*); void hipe_inc_nstack(Process *p); void hipe_empty_nstack(Process *p); -void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free); +void hipe_set_closure_stub(ErlFunEntry *fe); Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s); ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity); diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c new file mode 100644 index 0000000000..45d47272ed --- /dev/null +++ b/erts/emulator/hipe/hipe_module.c @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016-2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "hipe_arch.h" +#include "hipe_module.h" + +void hipe_free_module(HipeModule *mod) +{ + hipe_free_code(mod->text_segment, mod->text_segment_size); + if (mod->data_segment) /* Some modules lack data segments */ + erts_free(ERTS_ALC_T_HIPE_LL, mod->data_segment); + + erts_free(ERTS_ALC_T_HIPE_LL, mod); +} diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h new file mode 100644 index 0000000000..b489f567cb --- /dev/null +++ b/erts/emulator/hipe/hipe_module.h @@ -0,0 +1,45 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_module.h + * + * + */ +#ifndef HIPE_MODULE_H +#define HIPE_MODULE_H + +/* Forward-declare type to resolve circular dependency with module.h */ +typedef struct hipe_module HipeModule; + +#include "global.h" + +struct hipe_module { + void *text_segment; + Uint text_segment_size; + void *data_segment; + + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ +}; + +extern void hipe_free_module(HipeModule *mod); + +#endif /* HIPE_MODULE_H */ diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 9c03b3811c..d8044fe6da 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,8 +42,8 @@ */ /* for -Wmissing-prototypes :-( */ -extern Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2); -extern Eterm hipe_show_nstack_1(BIF_ALIST_1); +extern Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1); +extern Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1); /* Used when a BIF can trigger a stack walk. */ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) @@ -51,22 +51,24 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) p->hipe.narity = arity; } -Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1) { Eterm ret; - hipe_set_narity(BIF_P, 2); - ret = erts_internal_check_process_code_2(BIF_P, BIF__ARGS); + hipe_set_narity(BIF_P, 1); + ret = nbif_impl_erts_internal_check_process_code_1(NBIF_CALL_ARGS); hipe_set_narity(BIF_P, 0); return ret; } -Eterm hipe_show_nstack_1(BIF_ALIST_1) +/* Called via standard_bif_interface_1 */ +Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1) { Eterm ret; hipe_set_narity(BIF_P, 1); - ret = hipe_bifs_show_nstack_1(BIF_P, BIF__ARGS); + ret = nbif_impl_hipe_bifs_show_nstack_1(NBIF_CALL_ARGS); hipe_set_narity(BIF_P, 0); return ret; } @@ -89,7 +91,7 @@ void hipe_gc(Process *p, Eterm need) * has begun. * XXX: BUG: native code should check return status */ -BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1) +BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1) { Process* p = BIF_P; Eterm timeout_value = BIF_ARG_1; @@ -226,11 +228,6 @@ void hipe_handle_exception(Process *c_p) ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ - if (c_p->mbuf) { - erts_printf("%s line %u: p==%p, p->mbuf==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf); - /* erts_garbage_collect(c_p, 0, NULL, 0); */ - } - /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -259,11 +256,6 @@ void hipe_handle_exception(Process *c_p) /* Synthesized to avoid having to generate code for it. */ c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)]; - if (c_p->mbuf) { - /* erts_printf("%s line %u: p==%p, p->mbuf==%p, p->lastbif==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf, c_p->hipe.lastbif); */ - erts_garbage_collect(c_p, 0, NULL, 0); - } - hipe_find_handler(c_p); } @@ -280,10 +272,10 @@ static struct StackTrace *get_trace_from_exc(Eterm exc) * This does what the (misnamed) Beam instruction 'raise_ss' does, * namely, a proper re-throw of an exception that was caught by 'try'. */ - -BIF_RETTYPE hipe_rethrow(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2) { - Process* c_p = BIF_P; + Process *c_p = BIF_P; Eterm exc = BIF_ARG_1; Eterm value = BIF_ARG_2; @@ -331,7 +323,6 @@ char *hipe_bs_allocate(int len) Binary *bptr; bptr = erts_bin_nrml_alloc(len); - erts_smp_atomic_init_nob(&bptr->refc, 1); return bptr->orig_bytes; } @@ -407,7 +398,7 @@ Eterm hipe_bs_utf8_size(Eterm arg) return make_small(4); } -BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3) +BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3) { Process* p = BIF_P; Eterm arg = BIF_ARG_1; @@ -468,7 +459,7 @@ Eterm hipe_bs_put_utf16(Process *p, Eterm arg, byte *base, unsigned int offset, return new_offset; } -BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3) +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3) { Process *p = BIF_P; Eterm arg = BIF_ARG_1; @@ -477,7 +468,7 @@ BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3) return hipe_bs_put_utf16(p, arg, base, offset, 0); } -BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3) +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3) { Process *p = BIF_P; Eterm arg = BIF_ARG_1; @@ -495,7 +486,7 @@ static int validate_unicode(Eterm arg) return 1; } -BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1) +BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1) { Process *p = BIF_P; Eterm arg = BIF_ARG_1; @@ -513,7 +504,8 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg) return 1; } -BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2) { /* Arguments are Eterm-sized unsigned integers */ Uint dividend = BIF_ARG_1; diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h index a02d26087b..38f874888b 100644 --- a/erts/emulator/hipe/hipe_native_bif.h +++ b/erts/emulator/hipe/hipe_native_bif.h @@ -74,27 +74,27 @@ AEXTERN(void,nbif_select_msg,(Process*)); AEXTERN(Eterm,nbif_cmp_2,(void)); AEXTERN(Eterm,nbif_eq_2,(void)); -BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2); -BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1); +BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1); void hipe_fclearerror_error(Process*); void hipe_select_msg(Process*); void hipe_gc(Process*, Eterm); -BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1); +BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1); void hipe_handle_exception(Process*); -BIF_RETTYPE hipe_rethrow(BIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2); char *hipe_bs_allocate(int); Binary *hipe_bs_reallocate(Binary*, int); int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned); void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned); Eterm hipe_bs_utf8_size(Eterm); -BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3); +BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3); Eterm hipe_bs_utf16_size(Eterm); -BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3); -BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3); -BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1); +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3); +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3); +BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1); struct erl_bin_match_buffer; int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm); -BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2); #ifdef NO_FPE_SIGNALS AEXTERN(void,nbif_emulate_fpe,(Process*)); @@ -129,7 +129,7 @@ void hipe_atomic_inc(int*); void hipe_clear_timeout(Process*); #endif -#define BIF_LIST(M,F,A,C,I) AEXTERN(Eterm,nbif_##C,(void)); +#define BIF_LIST(M,F,A,B,C,I) AEXTERN(Eterm,nbif_##C,(void)); #include "erl_bif_list.h" #undef BIF_LIST diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..4413748936 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes) asm volatile("sync\n\tisync"); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -/* Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} - static int check_callees(Eterm callees) { Eterm *tuple; @@ -119,126 +90,71 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) + +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) { - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; + Uint32* trampoline = address; + int i; - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - Eterm mfa = tuple_val(callees)[trampnr]; - Eterm m = tuple_val(mfa)[1]; - Eterm f = tuple_val(mfa)[2]; - unsigned int a = unsigned_val(tuple_val(mfa)[3]); - unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a); - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { + for (i = 0; i < nrcallees; ++i) { #if defined(__powerpc64__) - if (nrfreewords < 7) - return NULL; - nrfreewords -= 7; - tramp_pos = trampoline = tramp_pos - 7; - trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ - trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ - trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ - trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[5] = 0x7D6903A6; /* mtctr r11 */ - trampoline[6] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 7*sizeof(int)); +# define TRAMPOLINE_WORDS 7 + trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ + trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ + trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ + trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[5] = 0x7D6903A6; /* mtctr r11 */ + trampoline[6] = 0x4E800420; /* bctr */ #else - if (nrfreewords < 4) - return NULL; - nrfreewords -= 4; - tramp_pos = trampoline = tramp_pos - 4; - trampoline[0] = 0x39600000; /* addi r11,r0,0 */ - trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ - trampoline[2] = 0x7D6903A6; /* mtctr r11 */ - trampoline[3] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 4*sizeof(int)); +# define TRAMPOLINE_WORDS 4 + trampoline[0] = 0x39600000; /* addi r11,r0,0 */ + trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ + trampoline[2] = 0x7D6903A6; /* mtctr r11 */ + trampoline[3] = 0x4E800420; /* bctr */ #endif - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); + + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); + + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } static void patch_imm16(Uint32 *address, unsigned int imm16) @@ -288,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(7); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32)); if (!code) return NULL; @@ -312,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[6] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 7*sizeof(int)); + hipe_flush_icache_range(code, 7*sizeof(Uint32)); return code; } @@ -360,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; /* * Native code calls BEAM via a stub looking as follows: @@ -383,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(4); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32)); if (!code) return NULL; @@ -396,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[3] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 4*sizeof(Uint32)); return code; } diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4 index 57b4208bee..79a8bef77d 100644 --- a/erts/emulator/hipe/hipe_ppc_bifs.m4 +++ b/erts/emulator/hipe/hipe_ppc_bifs.m4 @@ -26,9 +26,9 @@ include(`hipe/hipe_ppc_asm.m4') #`include' "hipe_literals.h" `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) STORE_IA(CSYM(F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) +# define CALL_BIF(F) STORE_IA(CSYM(nbif_impl_##F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) #else -# define CALL_BIF(F) bl CSYM(F) +# define CALL_BIF(F) bl CSYM(nbif_impl_##F) #endif' .text @@ -212,8 +212,9 @@ ASYM($1): * gc_bif_interface_0(nbif_name, cbif_name) * gc_bif_interface_1(nbif_name, cbif_name) * gc_bif_interface_2(nbif_name, cbif_name) + * gc_bif_interface_3(nbif_name, cbif_name) * - * Generate native interface for a BIF with 0-2 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. * The BIF may do a GC. */ @@ -300,6 +301,39 @@ ASYM($1): TYPE_FUNCTION(ASYM($1)) #endif') +define(gc_bif_interface_3, +` +#ifndef HAVE_$1 +#`define' HAVE_$1 + GLOBAL(ASYM($1)) +ASYM($1): + /* Set up C argument registers. */ + mr r3, P + NBIF_ARG(r4,3,0) + NBIF_ARG(r5,3,1) + NBIF_ARG(r6,3,2) + + /* Save caller-save registers and call the C function. */ + SAVE_CONTEXT_GC + STORE r4, P_ARG0(r3) /* Store BIF__ARGS in def_arg_reg[] */ + STORE r5, P_ARG1(r3) + STORE r6, P_ARG2(r3) + addi r4, r3, P_ARG0 + CALL_BIF($2) + TEST_GOT_MBUF + + /* Restore registers. Check for exception. */ + CMPI r3, THE_NON_VALUE + RESTORE_CONTEXT_GC + beq- 1f + NBIF_RET(3) +1: /* workaround for bc:s small offset operand */ + b CSYM(nbif_3_simple_exception) + HANDLE_GOT_MBUF(3) + SET_SIZE(ASYM($1)) + TYPE_FUNCTION(ASYM($1)) +#endif') + /* * gc_nofail_primop_interface_1(nbif_name, cbif_name) * diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h index 21c4239753..cc92bf653c 100644 --- a/erts/emulator/hipe/hipe_process.h +++ b/erts/emulator/hipe/hipe_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,6 +52,9 @@ struct hipe_process_state { #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) void (*bif_callee)(void); /* When calling BIF's via debug wrapper */ #endif +#ifdef DEBUG + UWord gc_is_unsafe; /* Nonzero when GC-required state is on stack */ +#endif }; extern void hipe_arch_print_pcb(struct hipe_process_state *p); @@ -68,12 +71,15 @@ static __inline__ void hipe_init_process(struct hipe_process_state *p) p->nra = NULL; #endif p->narity = 0; +#ifdef DEBUG + p->gc_is_unsafe = 0; +#endif } static __inline__ void hipe_delete_process(struct hipe_process_state *p) { if (p->nstack) - erts_free(ERTS_ALC_T_HIPE, (void*)p->nstack); + erts_free(ERTS_ALC_T_HIPE_STK, (void*)p->nstack); } #ifdef ERTS_SMP diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h index 315f8e7f9f..f019434f67 100644 --- a/erts/emulator/hipe/hipe_risc_gc.h +++ b/erts/emulator/hipe/hipe_risc_gc.h @@ -27,7 +27,7 @@ /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */ struct nstack_walk_state { - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ }; static inline int nstack_walk_init_check(const Process *p) @@ -43,15 +43,28 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) return p->hipe.nsp + nstkarity; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); state->sdesc0 = sdesc; return sdesc; } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline const struct hipe_sdesc* +nstack_walk_init_sdesc_ignore_trap(const Process *p, + struct nstack_walk_state *state) +{ + unsigned long ra = (unsigned long)p->hipe.nra; + const struct hipe_sdesc *sdesc; + if (ra == (unsigned long)&nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + sdesc = hipe_find_sdesc(ra); + state->sdesc0 = sdesc; + return sdesc; +} + +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { Eterm *nsp = p->hipe.nsp; p->hipe.nsp = nstack_walk_nsp_begin(p); @@ -90,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -101,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 0284265307..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,13 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .summary = (1<<8), - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index dc98c96b8f..4001bedeb6 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -237,7 +237,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; @@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; int i; if (depth < 1) @@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) ra = (unsigned long)p->hipe.nra; prev_ra = 0; i = 0; - for (;;) { + while (nsp < nsp_end) { if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; if (ra != prev_ra) { @@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) break; prev_ra = ra; } - if (nsp >= nsp_end) - break; sdesc = hipe_find_sdesc(ra); nsp += arity + sdesc_fsize(sdesc); arity = sdesc_arity(sdesc); diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 23020f34ee..876b20bb15 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -88,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint32 relDest, newI; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress); newI = (1 << 30) | (relDest >> 2); *(Uint32*)callAddress = newI; @@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int nrbytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; @@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4 index 2e886ec1d1..14330c2f1c 100644 --- a/erts/emulator/hipe/hipe_sparc_bifs.m4 +++ b/erts/emulator/hipe/hipe_sparc_bifs.m4 @@ -29,9 +29,9 @@ include(`hipe/hipe_sparc_asm.m4') .align 4 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) set F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper +# define CALL_BIF(F) set nbif_impl_##F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper #else -# define CALL_BIF(F) call F +# define CALL_BIF(F) call nbif_impl_##F #endif' /* @@ -210,8 +210,9 @@ $1: * gc_bif_interface_0(nbif_name, cbif_name) * gc_bif_interface_1(nbif_name, cbif_name) * gc_bif_interface_2(nbif_name, cbif_name) + * gc_bif_interface_3(nbif_name, cbif_name) * - * Generate native interface for a BIF with 0-2 parameters and + * Generate native interface for a BIF with 0-3 parameters and * standard failure mode. * The BIF may do a GC. */ @@ -295,6 +296,37 @@ $1: .type $1, #function #endif') +define(gc_bif_interface_3, +` +#ifndef HAVE_$1 +#`define' HAVE_$1 + .global $1 +$1: + /* Set up C argument registers. */ + mov P, %o0 + NBIF_ARG(%o1,3,0) + NBIF_ARG(%o2,3,1) + NBIF_ARG(%o3,3,2) + + /* Save caller-save registers and call the C function. */ + SAVE_CONTEXT_GC + st %o1, [%o0+P_ARG0] ! Store BIF__ARGS in def_arg_reg + st %o2, [%o0+P_ARG1] + st %o3, [%o0+P_ARG2] + add %o0, P_ARG0, %o1 + CALL_BIF($2) + nop + TEST_GOT_MBUF + + /* Restore registers. Check for exception. */ + TEST_GOT_EXN(3) + RESTORE_CONTEXT_GC + NBIF_RET(3) + HANDLE_GOT_MBUF(3) + .size $1, .-$1 + .type $1, #function +#endif') + /* * gc_nofail_primop_interface_1(nbif_name, cbif_name) * diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index e2e6eb74b1..4ee06690cf 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2016. All Rights Reserved. + * Copyright Ericsson AB 2003-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,10 +43,10 @@ */ struct hipe_sdesc_table hipe_sdesc_table; -static struct sdesc **alloc_bucket(unsigned int size) +static struct hipe_sdesc **alloc_bucket(unsigned int size) { - unsigned long nbytes = size * sizeof(struct sdesc*); - struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + unsigned long nbytes = size * sizeof(struct hipe_sdesc*); + struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE_LL, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size) static void hipe_grow_sdesc_table(void) { unsigned int old_size, new_size, new_mask; - struct sdesc **old_bucket, **new_bucket; + struct hipe_sdesc **old_bucket, **new_bucket; unsigned int i; old_size = 1 << hipe_sdesc_table.log2size; @@ -66,23 +66,23 @@ static void hipe_grow_sdesc_table(void) new_bucket = alloc_bucket(new_size); hipe_sdesc_table.bucket = new_bucket; for (i = 0; i < old_size; ++i) { - struct sdesc *b = old_bucket[i]; + struct hipe_sdesc *b = old_bucket[i]; while (b != NULL) { - struct sdesc *next = b->bucket.next; + struct hipe_sdesc *next = b->bucket.next; unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask; b->bucket.next = new_bucket[j]; new_bucket[j] = b; b = next; } } - erts_free(ERTS_ALC_T_HIPE, old_bucket); + erts_free(ERTS_ALC_T_HIPE_LL, old_bucket); } -struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) +struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) { unsigned long ra; unsigned int i; - struct sdesc *chain; + struct hipe_sdesc *chain; unsigned int size; ra = sdesc->bucket.hvalue; @@ -102,7 +102,29 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) return sdesc; } -void hipe_init_sdesc_table(struct sdesc *sdesc) +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE_LL, free_me); +} + +void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; - struct sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + struct hipe_sdesc *sdesc; + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(6) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -155,27 +192,34 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) } /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra - ? offsetof(struct sdesc_with_exnra, sdesc.livebits) - : offsetof(struct sdesc, livebits)) + ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits) + : offsetof(struct hipe_sdesc, livebits)) + livebitswords * sizeof(int); - p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes); + p = erts_alloc(ERTS_ALC_T_HIPE_LL, sdescbytes); /* If we have an exception handler use the special sdesc_with_exnra structure. */ if (exnra) { - struct sdesc_with_exnra *sdesc_we = p; + struct hipe_sdesc_with_exnra *sdesc_we = p; sdesc_we->exnra = exnra; sdesc = &(sdesc_we->sdesc); } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; - sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity; + sdesc->fsize = fsize; + sdesc->has_exnra = (exnra ? 1 : 0); + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -184,13 +228,5 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 4ea7d5c031..7e30358767 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -30,40 +30,43 @@ #include <stddef.h> /* offsetof() */ -struct sdesc { +struct hipe_sdesc { struct { unsigned long hvalue; /* return address */ - struct sdesc *next; /* hash collision chain */ + struct hipe_sdesc *next; /* hash collision chain */ } bucket; - unsigned int summary; /* frame size, exn handler presence flag, arity */ -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int fsize : 23; /* frame size */ + unsigned int has_exnra : 1; /* exn handler presence flag */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; -struct sdesc_with_exnra { +struct hipe_sdesc_with_exnra { unsigned long exnra; - struct sdesc sdesc; + struct hipe_sdesc sdesc; }; -static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { - return sdesc->summary >> 9; + return sdesc->fsize; } -static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc) +/* Nr of arguments pushed on stack */ +static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->summary & 0xFF; + return sdesc->stk_nargs; } -static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc) +static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { - if ((sdesc->summary & (1<<8))) { + if (sdesc->has_exnra) { const char *tmp; - tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc); - return ((const struct sdesc_with_exnra*)tmp)->exnra; + tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); + return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; } return 0; } @@ -72,13 +75,14 @@ struct hipe_sdesc_table { unsigned int log2size; unsigned int mask; /* INV: mask == (1 << log2size)-1 */ unsigned int used; - struct sdesc **bucket; + struct hipe_sdesc **bucket; }; extern struct hipe_sdesc_table hipe_sdesc_table; -extern struct sdesc *hipe_put_sdesc(struct sdesc*); -extern void hipe_init_sdesc_table(struct sdesc*); -extern struct sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); +extern void hipe_init_sdesc_table(struct hipe_sdesc*); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) @@ -86,10 +90,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra) +static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra) { unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; - const struct sdesc *sdesc = hipe_sdesc_table.bucket[i]; + const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i]; if (likely(sdesc->bucket.hvalue == ra)) return sdesc; do { @@ -103,7 +107,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void)); extern void hipe_print_nstack(Process*); extern void hipe_find_handler(Process*); extern void (*hipe_handle_stack_trap(Process*))(void); -extern void hipe_update_stack_trap(Process*, const struct sdesc*); +extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*); extern int hipe_fill_stacktrace(Process*, int, Eterm**); #if 0 && defined(HIPE_NSTACK_GROWS_UP) @@ -131,5 +135,10 @@ static __inline__ void hipe_check_nstack(Process *p, unsigned nwords) */ extern Eterm *fullsweep_nstack(Process *p, Eterm *n_htop); extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop); +extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area, + Uint area_size); +extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size); +extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size); + #endif /* HIPE_STACK_H */ diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..c7e24673ac 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -62,118 +61,17 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Uint)destAddress - (Uint)callAddress - 4; *(Uint32*)callAddress = rel32; hipe_flush_icache_word(callAddress); return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -/* FreeBSD 6.1 and Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -184,6 +82,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +167,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4 index b8ac5046d5..aecf67dc1b 100644 --- a/erts/emulator/hipe/hipe_x86_bifs.m4 +++ b/erts/emulator/hipe/hipe_x86_bifs.m4 @@ -32,9 +32,9 @@ include(`hipe/hipe_x86_asm.m4') #endif' `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) movl $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) +# define CALL_BIF(F) movl $CSYM(nbif_impl_##F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) #else -# define CALL_BIF(F) call CSYM(F) +# define CALL_BIF(F) call CSYM(nbif_impl_##F) #endif' define(TEST_GOT_MBUF,`movl P_MBUF(P), %edx /* `TEST_GOT_MBUF' */ @@ -666,15 +666,12 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu) #endif /* NO_FPE_SIGNALS */ /* - * Implement gc_bif_interface_0 as nofail_primop_interface_0. - */ -define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)') - -/* - * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2). + * Implement gc_bif_interface_N as standard_bif_interface_N. */ +define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)') define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)') define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)') +define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)') /* * Implement gc_nofail_primop_interface_1 as nofail_primop_interface_1. diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index c22b28c2d5..a703e24b8c 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -30,9 +30,9 @@ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ #else - struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ + struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ #endif }; @@ -57,31 +57,47 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); + const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity; + state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */ + state->sdesc0[0].fsize = 0; + state->sdesc0[0].has_exnra = 0; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; #endif } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline const struct hipe_sdesc* +nstack_walk_init_sdesc_ignore_trap(const Process *p, + struct nstack_walk_state *state) +{ +#ifdef SKIP_YOUNGEST_FRAME + unsigned long ra = p->hipe.nsp[0]; + const struct hipe_sdesc *sdesc; + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + sdesc = hipe_find_sdesc(ra); + state->sdesc0 = sdesc; + return sdesc; +#else + return nstack_walk_init_sdesc(p, state); +#endif +} + +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { #ifdef SKIP_YOUNGEST_FRAME Eterm *nsp = p->hipe.nsp; @@ -120,7 +136,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -131,7 +147,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 818d7444e2..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,16 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .summary = (1<<8), - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index 50d08b96d3..be68d7d463 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -2,7 +2,7 @@ * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2016. All Rights Reserved. + * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ #endif #include "hipe_signal.h" -#if __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) +#if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) /* * __libc_sigaction() is the core routine. * Without libpthread, sigaction() and __sigaction() are both aliases @@ -267,7 +267,7 @@ void hipe_thread_signal_init(void) { /* Stack don't really need to be cache aligned. We use it to suppress false leak report from valgrind */ - hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE, SIGSTKSZ)); + hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE_LL, SIGSTKSZ)); } #endif diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index f1559b1451..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -52,15 +52,14 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - struct sdesc sdesc0; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + struct hipe_sdesc sdesc0; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,10 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - sdesc0.summary = nstkarity; + sdesc0.fsize = 0; + sdesc0.has_exnra = 0; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; @@ -158,7 +157,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -199,7 +198,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -220,7 +219,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; nsp = p->hipe.nsp; @@ -262,7 +261,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; int i; |