From 9e9e7bf64fa97663dd4fb645c014880205fb46db Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 9 Jan 2017 15:13:58 +0100 Subject: erts: Add assertions for correct ErlNifEnv when constructing container terms. --- erts/emulator/beam/erl_nif.c | 107 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 4 deletions(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 5499512dd1..10514f6b3f 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -89,6 +89,14 @@ static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz); # define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0) #endif +#ifdef ERTS_NIF_ASSERT_IN_ENV +# define ASSERT_IN_ENV(ENV, TERM, NR, TYPE) dbg_assert_in_env(ENV, TERM, NR, TYPE, __func__) +static void dbg_assert_in_env(ErlNifEnv*, Eterm term, int nr, const char* type, const char* func); +# include "erl_gc.h" +#else +# define ASSERT_IN_ENV(ENV, TERM, NR, TYPE) +#endif + #ifdef DEBUG static int is_offheap(const ErlOffHeap* off_heap); #endif @@ -196,6 +204,9 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, ASSERT(p->common.id != ERTS_INVALID_PID); +#ifdef ERTS_NIF_ASSERT_IN_ENV + env->dbg_disable_assert_in_env = 0; +#endif #if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); @@ -474,11 +485,15 @@ setup_nif_env(struct enif_msg_environment_t* msg_env, HEAP_END(&msg_env->phony_proc) = phony_heap; MBUF(&msg_env->phony_proc) = NULL; msg_env->phony_proc.common.id = ERTS_INVALID_PID; + msg_env->env.tracee = tracee; + #ifdef FORCE_HEAP_FRAGS msg_env->phony_proc.space_verified = 0; msg_env->phony_proc.space_verified_from = NULL; #endif - msg_env->env.tracee = tracee; +#ifdef ERTS_NIF_ASSERT_IN_ENV + msg_env->env.dbg_disable_assert_in_env = 0; +#endif } ErlNifEnv* enif_alloc_env(void) @@ -1586,6 +1601,9 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt+1); Eterm ret = make_tuple(hp); va_list ap; @@ -1593,7 +1611,9 @@ ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) *hp++ = make_arityval(cnt); va_start(ap,cnt); while (cnt--) { - *hp++ = va_arg(ap,Eterm); + Eterm elem = va_arg(ap,Eterm); + ASSERT_IN_ENV(env, elem, ++nr, "tuple"); + *hp++ = elem; } va_end(ap); return ret; @@ -1601,12 +1621,16 @@ ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) ERL_NIF_TERM enif_make_tuple_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt) { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt+1); Eterm ret = make_tuple(hp); const Eterm* src = arr; *hp++ = make_arityval(cnt); while (cnt--) { + ASSERT_IN_ENV(env, *src, ++nr, "tuple"); *hp++ = *src++; } return ret; @@ -1617,6 +1641,8 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr) Eterm* hp = alloc_heap(env,2); Eterm ret = make_list(hp); + ASSERT_IN_ENV(env, car, 0, "head of list cell"); + ASSERT_IN_ENV(env, cdr, 0, "tail of list cell"); CAR(hp) = car; CDR(hp) = cdr; return ret; @@ -1628,6 +1654,9 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) return NIL; } else { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt*2); Eterm ret = make_list(hp); Eterm* last = &ret; @@ -1635,8 +1664,10 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) va_start(ap,cnt); while (cnt--) { + Eterm term = va_arg(ap,Eterm); *last = make_list(hp); - *hp = va_arg(ap,Eterm); + ASSERT_IN_ENV(env, term, ++nr, "list"); + *hp = term; last = ++hp; ++hp; } @@ -1648,14 +1679,19 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt) { +#ifdef ERTS_NIF_ASSERT_IN_ENV + int nr = 0; +#endif Eterm* hp = alloc_heap(env,cnt*2); Eterm ret = make_list(hp); Eterm* last = &ret; const Eterm* src = arr; while (cnt--) { + Eterm term = *src++; *last = make_list(hp); - *hp = *src++; + ASSERT_IN_ENV(env, term, ++nr, "list"); + *hp = term; last = ++hp; ++hp; } @@ -2789,6 +2825,10 @@ int enif_make_map_put(ErlNifEnv* env, if (!is_map(map_in)) { return 0; } + ASSERT_IN_ENV(env, map_in, 0, "old map"); + ASSERT_IN_ENV(env, key, 0, "key"); + ASSERT_IN_ENV(env, value, 0, "value"); + flush_env(env); *map_out = erts_maps_put(env->proc, key, value, map_in); cache_env(env); @@ -2823,6 +2863,10 @@ int enif_make_map_update(ErlNifEnv* env, return 0; } + ASSERT_IN_ENV(env, map_in, 0, "old map"); + ASSERT_IN_ENV(env, key, 0, "key"); + ASSERT_IN_ENV(env, value, 0, "value"); + flush_env(env); res = erts_maps_update(env->proc, key, value, map_in, map_out); cache_env(env); @@ -3543,6 +3587,9 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, clear_offheap(&MSO(p)); erts_pre_nif(&env, p, mod, tracee); +#ifdef ERTS_NIF_ASSERT_IN_ENV + env.dbg_disable_assert_in_env = 1; +#endif nif_result = (*fun->fptr)(&env, argc, argv); if (env.exception_thrown) nif_result = THE_NON_VALUE; @@ -3565,6 +3612,9 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, so we create a phony one. */ struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, mod, tracee); +#ifdef ERTS_NIF_ASSERT_IN_ENV + msg_env.env.dbg_disable_assert_in_env = 1; +#endif nif_result = (*fun->fptr)(&msg_env.env, argc, argv); if (msg_env.env.exception_thrown) nif_result = THE_NON_VALUE; @@ -3629,6 +3679,55 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size) #endif /* READONLY_CHECK */ +#ifdef ERTS_NIF_ASSERT_IN_ENV +static void dbg_assert_in_env(ErlNifEnv* env, Eterm term, + int nr, const char* type, const char* func) +{ + Uint saved_used_size; + Eterm* real_htop; + + if (is_immed(term) + || (is_non_value(term) && env->exception_thrown) + || erts_is_literal(term, ptr_val(term))) + return; + + if (env->dbg_disable_assert_in_env) { + /* + * Trace nifs may cheat as built terms are discarded after return. + * ToDo: Check if 'term' is part of argv[]. + */ + return; + } + + if (env->heap_frag) { + ASSERT(env->heap_frag == MBUF(env->proc)); + ASSERT(env->hp >= env->heap_frag->mem); + ASSERT(env->hp <= env->heap_frag->mem + env->heap_frag->alloc_size); + saved_used_size = env->heap_frag->used_size; + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + real_htop = NULL; + } + else { + real_htop = env->hp; + } + if (!erts_dbg_within_proc(ptr_val(term), env->proc, real_htop)) { + fprintf(stderr, "\r\nFAILED ASSERTION in %s:\r\n", func); + if (nr) { + fprintf(stderr, "Term #%d of the %s is not from same ErlNifEnv.", + nr, type); + } + else { + fprintf(stderr, "The %s is not from the same ErlNifEnv.", type); + } + fprintf(stderr, "\r\nABORTING\r\n"); + abort(); + } + if (env->heap_frag) { + env->heap_frag->used_size = saved_used_size; + } +} +#endif + #ifdef HAVE_USE_DTRACE #define MESSAGE_BUFSIZ 1024 -- cgit v1.2.3 From 10ace079fb6d0d626a16a5c6726c6bb8e6bb3878 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 21 Dec 2016 17:43:28 +0100 Subject: erts: Cleanup enif_make_reverse_list --- erts/emulator/beam/erl_nif.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 10514f6b3f..cd44eb880e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1724,13 +1724,9 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size) driver_system_info(sip, si_size); } -int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) { - Eterm *listptr, ret = NIL, *hp; - - if (is_nil(term)) { - *list = term; - return 1; - } +int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) +{ + Eterm *listptr, ret, *hp; ret = NIL; -- cgit v1.2.3 From d74d7a4cc0808e1f6e710e5d17ec55bfe093cdd9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Jan 2017 18:19:30 +0100 Subject: erts: Fix enif_send from noproc and no msg_env Only try direct allocation on receiver heap when called in a real process context to avoid trylock on our own main lock. --- erts/emulator/beam/erl_nif.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6b265a8b80..dd4b88e4a3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -690,7 +690,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Uint sz = size_object(msg); ErlOffHeap *ohp; Eterm *hp; - if (env && !env->tracee) { + if (c_p && !env->tracee) { full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); full_cache_env(env); -- cgit v1.2.3 From bfcf88311828b93a833ce96ad1a518b8eca08552 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 18 Jan 2017 14:40:25 +0100 Subject: Change exception for enif_schedule_nif() with dirty flags --- erts/emulator/beam/erl_nif.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b860759fa2..af3ca3afa3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2506,10 +2506,13 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, if (flags == 0) result = schedule(env, execute_nif, fp, proc->current->module, fun_name_atom, argc, argv); + else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) { #ifdef ERTS_DIRTY_SCHEDULERS - else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); +#else + result = enif_raise_exception(env, am_notsup); #endif + } else result = enif_make_badarg(env); -- cgit v1.2.3 From b079018e38272604ffacfece9b97924a9e39df5c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 17:10:18 +0100 Subject: Implement magic references Magic references are *intentionally* indistinguishable from ordinary references for the Erlang software. Magic references do not change the language, and are intended as a pure runtime internal optimization. An ordinary reference is typically used as a key in some table. A magic reference has a direct pointer to a reference counted magic binary. This makes it possible to implement various things without having to do lookups in a table, but instead access the data directly. Besides very fast lookups this can also improve scalability by removing a potentially contended table. A couple of examples of planned future usage of magic references are ETS table identifiers, and BIF timer identifiers. Besides future optimizations using magic references it should also be possible to replace the exposed magic binary cludge with magic references. That is, magic binaries that are exposed as empty binaries to the Erlang software. --- erts/emulator/beam/erl_nif.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 7c7a32f234..20b00bd4ba 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1721,7 +1721,7 @@ ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string, ERL_NIF_TERM enif_make_ref(ErlNifEnv* env) { - Eterm* hp = alloc_heap(env, REF_THING_SIZE); + Eterm* hp = alloc_heap(env, ERTS_REF_THING_SIZE); return erts_make_ref_in_buffer(hp); } @@ -1967,7 +1967,6 @@ typedef struct enif_resource_t byte align__[4]; # endif #endif - char data[1]; }ErlNifResource; @@ -2166,6 +2165,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), &nif_resource_dtor, + ERTS_ALC_T_BINARY, 1); /* unaligned */ ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); -- cgit v1.2.3 From 7fee9c57d89be05c980d1684329b26ac991bb26a Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 23 Jan 2017 17:16:28 +0100 Subject: Use magic refs for NIF resources --- erts/emulator/beam/erl_nif.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 20b00bd4ba..1de56ce0a1 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2209,34 +2209,40 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); - Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); - return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); + Eterm* hp = alloc_heap(env, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(env->proc), &bin->binary); } ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, const void* data, size_t size) { - Eterm bin = enif_make_resource(env, obj); - ProcBin* pb = (ProcBin*) binary_val(bin); + ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); + Eterm ebin = erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); + ProcBin* pb = (ProcBin*) binary_val(ebin); pb->bytes = (byte*) data; pb->size = size; - return bin; + return ebin; } int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp) { - ProcBin* pb; Binary* mbin; ErlNifResource* resource; - if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { - return 0; + if (is_internal_magic_ref(term)) + mbin = erts_magic_ref2bin(term); + else { + ProcBin* pb; + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) + return 0; + pb = (ProcBin*) binary_val(term); + /*if (pb->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + }*/ + mbin = pb->val; } - pb = (ProcBin*) binary_val(term); - /*if (pb->size != 0) { - return 0; / * Or should we allow "resource binaries" as handles? * / - }*/ - mbin = pb->val; resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { -- cgit v1.2.3 From 28735b9c2c6390df593b05f300151addbd01e367 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 24 Jan 2017 17:31:53 +0100 Subject: Adjust the only usage of exposed magic binaries --- erts/emulator/beam/erl_nif.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 1de56ce0a1..3674a29bf8 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2218,12 +2218,22 @@ ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); + ErlOffHeap *ohp = &MSO(env->proc); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); - Eterm ebin = erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); - ProcBin* pb = (ProcBin*) binary_val(ebin); - pb->bytes = (byte*) data; + ProcBin* pb = (ProcBin *) hp; + + pb->thing_word = HEADER_PROC_BIN; pb->size = size; - return ebin; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*) pb; + pb->val = &bin->binary; + pb->bytes = (byte*) data; + pb->flags = 0; + + OH_OVERHEAD(ohp, size / sizeof(Eterm)); + erts_refc_inc(&bin->binary.refc, 1); + + return make_binary(hp); } int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, @@ -2234,14 +2244,20 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ if (is_internal_magic_ref(term)) mbin = erts_magic_ref2bin(term); else { - ProcBin* pb; - if (!ERTS_TERM_IS_MAGIC_BINARY(term)) - return 0; - pb = (ProcBin*) binary_val(term); - /*if (pb->size != 0) { - return 0; / * Or should we allow "resource binaries" as handles? * / - }*/ - mbin = pb->val; + Eterm *hp; + if (!is_binary(term)) + return 0; + hp = binary_val(term); + if (thing_subtag(*hp) != REFC_BINARY_SUBTAG) + return 0; + /* + if (((ProcBin *) hp)->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + } + */ + mbin = ((ProcBin *) hp)->val; + if (!(mbin->flags & BIN_FLAG_MAGIC)) + return 0; } resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor -- cgit v1.2.3 From 118de47d703e303aea7f4575849a37c11416ba14 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 14 Feb 2017 18:26:31 +0100 Subject: erts: Add deallocation veto for magic destructors A magic destructor can return 0 and thereby take control and prolong the lifetime of a magic binary. --- erts/emulator/beam/erl_nif.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_nif.c') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3674a29bf8..61ae57a7f0 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -2140,7 +2140,7 @@ static void rollback_opened_resource_types(void) } -static void nif_resource_dtor(Binary* bin) +static int nif_resource_dtor(Binary* bin) { ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; @@ -2159,6 +2159,7 @@ static void nif_resource_dtor(Binary* bin) steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } + return 1; } void* enif_alloc_resource(ErlNifResourceType* type, size_t size) -- cgit v1.2.3