diff options
Diffstat (limited to 'erts/emulator/beam/erl_db_util.c')
-rw-r--r-- | erts/emulator/beam/erl_db_util.c | 1500 |
1 files changed, 825 insertions, 675 deletions
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 713ac0ba18..6732b708a8 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2016. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,6 +36,7 @@ #include "bif.h" #include "big.h" #include "erl_binary.h" +#include "erl_map.h" #include "erl_thr_progress.h" #include "erl_db_util.h" @@ -121,6 +123,9 @@ do { \ #define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm)))) +#define add_dmc_err(EINFO, STR, VAR, TERM, SEV) \ + vadd_dmc_err(EINFO, SEV, VAR, STR, TERM) + static ERTS_INLINE Process * get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) @@ -133,21 +138,22 @@ get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) static Eterm -set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) { +set_tracee_flags(Process *tracee_p, ErtsTracer tracer, + Uint d_flags, Uint e_flags) { Eterm ret; Uint flags; - if (tracer == NIL) { + if (ERTS_TRACER_IS_NIL(tracer)) { flags = ERTS_TRACE_FLAGS(tracee_p) & ~TRACEE_FLAGS; } else { flags = ((ERTS_TRACE_FLAGS(tracee_p) & ~d_flags) | e_flags); - if (! flags) tracer = NIL; + if (! flags) tracer = erts_tracer_nil; } - ret = ((ERTS_TRACER_PROC(tracee_p) != tracer + ret = ((!ERTS_TRACER_COMPARE(ERTS_TRACER(tracee_p),tracer) || ERTS_TRACE_FLAGS(tracee_p) != flags) ? am_true : am_false); - ERTS_TRACER_PROC(tracee_p) = tracer; + erts_tracer_replace(&tracee_p->common, tracer); ERTS_TRACE_FLAGS(tracee_p) = flags; return ret; } @@ -161,46 +167,18 @@ set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) { ** returns fail_term on failure. Fails if tracer pid or port is invalid. */ static Eterm -set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer, +set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer, Uint d_flags, Uint e_flags) { - Eterm ret = fail_term; - Process *tracer_p; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCKS_ALL == - erts_proc_lc_my_proc_locks(tracee_p)); - - if (is_internal_pid(tracer) - && (tracer_p = - erts_pid2proc(tracee_p, ERTS_PROC_LOCKS_ALL, - tracer, ERTS_PROC_LOCKS_ALL))) { - if (tracee_p != tracer_p) { - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - ERTS_TRACE_FLAGS(tracer_p) |= (ERTS_TRACE_FLAGS(tracee_p) - ? F_TRACER - : 0); - erts_smp_proc_unlock(tracer_p, ERTS_PROC_LOCKS_ALL); - } - } else if (is_internal_port(tracer)) { - Port *tracer_port = - erts_id2port_sflgs(tracer, - tracee_p, - ERTS_PROC_LOCKS_ALL, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (tracer_port) { - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - erts_port_release(tracer_port); - } - } else { - ASSERT(is_nil(tracer)); - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - } - return ret; -} + ERTS_SMP_LC_ASSERT( + ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p) + || erts_thr_progress_is_blocking()); -/* Type checking... */ - -#define BOXED_IS_TUPLE(Boxed) is_arity_value(*boxed_val((Boxed))) + if (ERTS_TRACER_IS_NIL(tracer) + || erts_is_tracer_enabled(tracer, &tracee_p->common)) + return set_tracee_flags(tracee_p, tracer, d_flags, e_flags); + return fail_term; +} /* ** @@ -217,7 +195,9 @@ typedef enum { matchTuple, matchPushT, matchPushL, + matchPushM, matchPop, + matchSwap, matchBind, matchCmp, matchEqBin, @@ -226,21 +206,21 @@ typedef enum { matchEqRef, matchEq, matchList, + matchMap, + matchKey, matchSkip, matchPushC, matchConsA, /* Car is below Cdr */ matchConsB, /* Cdr is below Car (unusual) */ matchMkTuple, + matchMkFlatMap, + matchMkHashMap, matchCall0, matchCall1, matchCall2, matchCall3, matchPushV, -#if HALFWORD_HEAP - matchPushVGuard, /* First guard-only variable reference */ -#endif - matchPushVResult, /* First variable reference in result, or (if HALFWORD) - in guard if also referenced in result */ + matchPushVResult, /* First variable reference in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -307,9 +287,6 @@ DMC_DECLARE_STACK_TYPE(unsigned); typedef struct DMCVariable { int is_bound; int is_in_body; -#if HALFWORD_HEAP - int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ -#endif } DMCVariable; typedef struct DMCHeap { @@ -371,7 +348,6 @@ typedef struct MatchVariable { Eterm term; #ifdef DEBUG Process* proc; - Eterm* base; #endif } MatchVariable; @@ -412,7 +388,7 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) else { int i; for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef; #else mpsp->default_heap[i] = (Eterm) 0xdeadbeef; @@ -439,17 +415,27 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) { ErtsMatchPseudoProcess *mpsp; #ifdef ERTS_SMP - mpsp = (ErtsMatchPseudoProcess *) c_p->scheduler_data->match_pseudo_process; - if (mpsp) + ErtsSchedulerData *esdp; + + esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data(); + + mpsp = esdp ? esdp->match_pseudo_process : + (ErtsMatchPseudoProcess*) erts_smp_tsd_get(match_pseudo_process_key); + + if (mpsp) { + ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); + ASSERT(mpsp->process.scheduler_data == esdp); cleanup_match_pseudo_process(mpsp, 0); + } else { ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL); mpsp = create_match_pseudo_process(); - c_p->scheduler_data->match_pseudo_process = (void *) mpsp; + if (esdp) { + esdp->match_pseudo_process = (void *) mpsp; + } + mpsp->process.scheduler_data = esdp; erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp); } - ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); - mpsp->process.scheduler_data = c_p->scheduler_data; #else mpsp = match_pseudo_process; cleanup_match_pseudo_process(mpsp, 0); @@ -482,7 +468,8 @@ void match_pseudo_process_init(void) { #ifdef ERTS_SMP - erts_smp_tsd_key_create(&match_pseudo_process_key); + erts_smp_tsd_key_create(&match_pseudo_process_key, + "erts_match_pseudo_process_key"); erts_smp_install_exit_handler(destroy_match_pseudo_process); #else match_pseudo_process = create_match_pseudo_process(); @@ -565,6 +552,12 @@ static DMCGuardBif guard_tab[] = DBIF_ALL }, { + am_is_map, + &is_map_1, + 1, + DBIF_ALL + }, + { am_is_binary, &is_binary_1, 1, @@ -631,6 +624,12 @@ static DMCGuardBif guard_tab[] = DBIF_ALL }, { + am_map_size, + &map_size_1, + 1, + DBIF_ALL + }, + { am_bit_size, &bit_size_1, 1, @@ -842,6 +841,13 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info); static Uint my_size_object(Eterm t); static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap); +/* Guard subroutines */ +static void +dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text, + int textpos, Eterm *p, Uint nelems); +static DMCRet +dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm *p, Uint nelems, int *constant); /* Guard compilation */ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t); @@ -855,6 +861,9 @@ static DMCRet dmc_tuple(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); +static DMCRet +dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant); static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -874,12 +883,14 @@ static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, Eterm t); -/* match expression subroutine */ +/* match expression subroutines */ static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, DMC_STACK_TYPE(UWord) *text, Eterm c); +static Eterm +dmc_private_copy(DMCContext *context, Eterm c); #ifdef DMC_DEBUG @@ -892,11 +903,7 @@ void db_match_dis(Binary *prog); #define TRACE /* Nothing */ #define FENCE_PATTERN_SIZE 0 #endif -static void add_dmc_err(DMCErrInfo *err_info, - char *str, - int variable, - Eterm term, - DMCErrorSeverity severity); +static void vadd_dmc_err(DMCErrInfo*, DMCErrorSeverity, int var, const char *str, ...); static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity); @@ -992,12 +999,20 @@ Eterm erts_match_set_get_source(Binary *mpsp) } /* This one is for the tracing */ -Binary *erts_match_set_compile(Process *p, Eterm matchexpr) { +Binary *erts_match_set_compile(Process *p, Eterm matchexpr, Eterm MFA) { Binary *bin; Uint sz; Eterm *hp; + Uint flags; + + switch (MFA) { + case am_receive: flags = DCOMP_TRACE; break; + case am_send: flags = DCOMP_TRACE | DCOMP_ALLOW_TRACE_OPS; break; + default: + flags = DCOMP_TRACE | DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS; + } - bin = db_match_set_compile(p, matchexpr, DCOMP_TRACE); + bin = db_match_set_compile(p, matchexpr, flags); if (bin != NULL) { MatchProg *prog = Binary2MatchProg(bin); sz = size_object(matchexpr); @@ -1127,8 +1142,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) int i; if (!is_list(matchexpr)) { - add_dmc_err(err_info, "Match programs are not in a list.", - -1, 0UL, dmcError); + add_dmc_err(err_info, "Match programs are not in a list.", + -1, 0UL, dmcError); goto done; } num_heads = 0; @@ -1136,9 +1151,8 @@ Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags) ++num_heads; if (l != NIL) { /* proper list... */ - add_dmc_err(err_info, "Match programs are not in a proper " - "list.", - -1, 0UL, dmcError); + add_dmc_err(err_info, "Match programs are not in a proper list.", + -1, 0UL, dmcError); goto done; } @@ -1205,30 +1219,37 @@ done: return ret; } -Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, - enum erts_pam_run_flags in_flags, - Uint32 *return_flags) +/* Returns + * am_false if no match or + * if {message,false} has been called, + * am_true if {message,_} has NOT been called or + * if {message,true} has been called, + * Msg if {message,Msg} has been called. + * + * If return value is_not_immed + * then erts_match_set_release_result_trace() must be called to release it. + */ +Eterm erts_match_set_run_trace(Process *c_p, + Process *self, + Binary *mpsp, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, + Uint32 *return_flags) { Eterm ret; - ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + ret = db_prog_match(c_p, self, mpsp, NIL, args, num_args, in_flags, return_flags); -#if defined(HARDDEBUG) - if (is_non_value(ret)) { - erts_fprintf(stderr, "Failed\n"); - } else { - erts_fprintf(stderr, "Returning : %T\n", ret); + + ASSERT(!(is_non_value(ret) && *return_flags)); + + if (is_non_value(ret) || ret == am_false) { + erts_match_set_release_result(c_p); + return am_false; } -#endif + if (is_immed(ret)) + erts_match_set_release_result(c_p); return ret; - /* Returns - * THE_NON_VALUE if no match - * am_false if {message,false} has been called, - * am_true if {message,_} has not been called or - * if {message,true} has been called, - * Msg if {message,Msg} has been called. - */ } static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, @@ -1237,7 +1258,8 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ret = db_prog_match(p, p, + mpsp, args, NULL, num_args, ERTS_PAM_COPY_RESULT, return_flags); #if defined(HARDDEBUG) @@ -1350,7 +1372,112 @@ restart: for (;;) { switch (t & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(t)) { + if (is_flatmap(t)) { + num_iters = flatmap_get_size(flatmap_val(t)); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + for (i = 0; i < num_iters; ++i) { + Eterm key = flatmap_get_keys(flatmap_val(t))[i]; + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + Eterm value = flatmap_get_values(flatmap_val(t))[i]; + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + break; + } + if (is_hashmap(t)) { + DECLARE_WSTACK(wstack); + Eterm *kv; + num_iters = hashmap_size(t); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + + hashmap_iterator_init(&wstack, t, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + Eterm key = CAR(kv); + Eterm value = CDR(kv); + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + DESTROY_WSTACK(wstack); + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + DESTROY_WSTACK(wstack); + break; + } + if (!is_tuple(t)) { goto simple_term; } num_iters = arityval(*tuple_val(t)); @@ -1622,96 +1749,31 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) return ret; } - -#if HALFWORD_HEAP -struct heap_checkpoint_t -{ - Process *p; - Eterm* htop; - ErlHeapFragment* mbuf; - unsigned used_size; - ErlOffHeap off_heap; -}; - -static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) -{ - hcp->p = p; - hcp->htop = HEAP_TOP(p); - hcp->mbuf = MBUF(p); - hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; - hcp->off_heap = MSO(p); -} - -static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) -{ - struct erl_off_heap_header* oh = MSO(hcp->p).first; - - if (oh != hcp->off_heap.first) { - ASSERT(oh != NULL); - if (hcp->off_heap.first) { - while (oh->next != hcp->off_heap.first) { - oh = oh->next; - } - oh->next = NULL; - } - erts_cleanup_offheap(&MSO(hcp->p)); - MSO(hcp->p) = hcp->off_heap; - } - if (MBUF(hcp->p) != hcp->mbuf) { - ErlHeapFragment* hf = MBUF(hcp->p); - ASSERT(hf != NULL); - if (hcp->mbuf) { - while (hf->next != hcp->mbuf) { - hf = hf->next; - } - hf->next = NULL; - } - free_message_buffer(MBUF(hcp->p)); - MBUF(hcp->p) = hcp->mbuf; - } - if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { - hcp->mbuf->used_size = hcp->used_size; - } - HEAP_TOP(hcp->p) = hcp->htop; -} -#endif /* HALFWORD_HEAP */ - -static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) -{ - if (!is_immed(term)) { - Uint sz = size_object_rel(term, base); - Eterm* top = HAllocX(p, sz, HEAP_XTRA); - return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); - } - return term; -} - - /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. ** the parameter 'arity' is only used if 'term' is actually an array, ** i.e. 'DCOMP_TRACE' was specified */ -Eterm db_prog_match(Process *c_p, Binary *bprog, - Eterm term, Eterm* base, +Eterm db_prog_match(Process *c_p, + Process *self, + Binary *bprog, + Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags) { MatchProg *prog = Binary2MatchProg(bprog); - Eterm *ep; - Eterm *tp; + const Eterm *ep, *tp, **sp; Eterm t; - Eterm **sp; Eterm *esp; MatchVariable* variables; BeamInstr *cp; - UWord *pc = prog->text; + const UWord *pc = prog->text; Eterm *ehp; Eterm ret; - Uint n = 0; /* To avoid warning. */ + Uint n; int i; unsigned do_catch; ErtsMatchPseudoProcess *mpsp; @@ -1723,17 +1785,16 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm (*bif)(Process*, ...); Eterm bif_args[3]; int fail_label; - int atomic_trace; -#if HALFWORD_HEAP - struct heap_checkpoint_t c_p_checkpoint = {}; -#endif #ifdef DMC_DEBUG Uint *heap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ - ASSERT(base==NULL || HALFWORD_HEAP); + ERTS_UNDEF(n,0); + ERTS_UNDEF(current_scheduled,NULL); + + ASSERT(c_p || !(in_flags & ERTS_PAM_COPY_RESULT)); mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1741,33 +1802,11 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, /* We need to lure the scheduler into believing in the pseudo process, because of floating point exceptions. Do *after* mpsp is set!!! */ - esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p); - ASSERT(esdp != NULL); - current_scheduled = esdp->current_process; + esdp = erts_get_scheduler_data(); + if (esdp) + current_scheduled = esdp->current_process; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ - atomic_trace = 0; -#define BEGIN_ATOMIC_TRACE(p) \ - do { \ - if (! atomic_trace) { \ - erts_refc_inc(&bprog->refc, 2); \ - erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ - erts_smp_thr_progress_block(); \ - atomic_trace = !0; \ - } \ - } while (0) -#define END_ATOMIC_TRACE(p) \ - do { \ - if (atomic_trace) { \ - erts_smp_thr_progress_unblock(); \ - erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ - if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ - erts_bin_free(bprog); \ - } \ - atomic_trace = 0; \ - } \ - } while (0) - #ifdef DMC_DEBUG save_op = 0; heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1; @@ -1785,29 +1824,23 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, do_catch != 0 */ *return_flags = 0U; - variables = mpsp->u.variables; -#if HALFWORD_HEAP - c_p_checkpoint.p = NULL; -#endif restart: ep = &term; esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset); - sp = (Eterm **) esp; + sp = (const Eterm **)esp; ret = am_true; do_catch = 0; fail_label = -1; build_proc = psp; esdp->current_process = psp; - ASSERT_HALFWORD(!c_p_checkpoint.p); #ifdef DEBUG ASSERT(variables == mpsp->u.variables); for (i=0; i<prog->num_bindings; i++) { variables[i].term = THE_NON_VALUE; variables[i].proc = NULL; - variables[i].base = base; } #endif @@ -1815,11 +1848,11 @@ restart: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -1838,14 +1871,14 @@ restart: ep = termp; break; case matchArrayBind: /* When the array size is unknown. */ - ASSERT(termp); + ASSERT(termp || arity==0); n = *pc++; variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - ep = tuple_val_rel(*ep,base); + ep = tuple_val(*ep); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1853,9 +1886,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - tp = tuple_val_rel(*ep,base); + tp = tuple_val(*ep); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1865,46 +1898,94 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val_rel(*ep,base); + ep = list_val(*ep); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val_rel(*ep,base); + *sp++ = list_val(*ep); ++ep; break; + case matchMap: + if (!is_map(*ep)) { + FAIL(); + } + n = *pc++; + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { + FAIL(); + } + } + ep = flatmap_val(*ep); + break; + case matchPushM: + if (!is_map(*ep)) { + FAIL(); + } + n = *pc++; + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { + FAIL(); + } + } + *sp++ = flatmap_val(*ep++); + break; + case matchKey: + t = (Eterm) *pc++; + tp = erts_maps_get(t, make_boxed(ep)); + if (!tp) { + FAIL(); + } + *sp++ = ep; + ep = tp; + break; case matchPop: ep = *(--sp); break; + case matchSwap: + tp = sp[-1]; + sp[-1] = sp[-2]; + sp[-2] = tp; + break; case matchBind: n = *pc++; variables[n].term = *ep++; break; case matchCmp: n = *pc++; - if (!eq_rel(variables[n].term, base, *ep, base)) + if (!EQ(variables[n].term, *ep)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq_rel(t,NULL,*ep,base)) + if (!EQ(t,*ep)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float_rel(*ep,base)) + if (!is_float(*ep)) FAIL(); - if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) + if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; break; case matchEqRef: { Eterm* epc = (Eterm*)pc; - if (!is_ref_rel(*ep,base)) + if (!is_ref(*ep)) FAIL(); - if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { + if (!EQ(make_internal_ref(epc), *ep)) { FAIL(); } i = thing_arityval(*epc); @@ -1913,9 +1994,9 @@ restart: break; } case matchEqBig: - if (!is_big_rel(*ep,base)) + if (!is_big(*ep)) FAIL(); - tp = big_val_rel(*ep,base); + tp = big_val(*ep); { Eterm *epc = (Eterm *) pc; if (*tp != *epc) @@ -1973,6 +2054,39 @@ restart: } *esp++ = t; break; + case matchMkFlatMap: + n = *pc++; + ehp = HAllocX(build_proc, MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA); + t = *--esp; + { + flatmap_t *m = (flatmap_t *)ehp; + m->thing_word = MAP_HEADER_FLATMAP; + m->size = n; + m->keys = t; + } + t = make_flatmap(ehp); + ehp += MAP_HEADER_FLATMAP_SZ; + while (n--) { + *ehp++ = *--esp; + } + *esp++ = t; + break; + case matchMkHashMap: + n = *pc++; + esp -= 2*n; + ehp = HAllocX(build_proc, 2*n, HEAP_XTRA); + { + ErtsHeapFactory factory; + Uint ix; + for (ix = 0; ix < 2*n; ix++){ + ehp[ix] = esp[ix]; + } + erts_factory_proc_init(&factory, build_proc); + t = erts_hashmap_from_array(&factory, ehp, n, 0); + erts_factory_close(&factory); + } + *esp++ = t; + break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; t = (*bif)(build_proc, bif_args); @@ -2024,62 +2138,36 @@ restart: esp -= 2; esp[-1] = t; break; - - #if HALFWORD_HEAP - case matchPushVGuard: - if (!base) goto case_matchPushV; - /* Build NULL-based copy on pseudo heap for easy disposal */ - n = *pc++; - ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(psp, variables[n].term, base); - *esp++ = variables[n].term; - #ifdef DEBUG - variables[n].proc = psp; - variables[n].base = NULL; - #endif - break; - #endif case matchPushVResult: if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; - - /* Build (NULL-based) copy on callers heap */ - #if HALFWORD_HEAP - if (!do_catch && !c_p_checkpoint.p) { - heap_checkpoint_init(c_p, &c_p_checkpoint); - } - #endif + /* Build copy on callers heap */ n = *pc++; ASSERT(is_value(variables[n].term)); ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(c_p, variables[n].term, base); + variables[n].term = copy_object_x(variables[n].term, c_p, HEAP_XTRA); *esp++ = variables[n].term; #ifdef DEBUG variables[n].proc = c_p; - variables[n].base = NULL; #endif break; case matchPushV: case_matchPushV: n = *pc++; ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].base); *esp++ = variables[n].term; break; case matchPushExpr: if (in_flags & ERTS_PAM_COPY_RESULT) { Uint sz; Eterm* top; - sz = size_object_rel(term, base); + sz = size_object(term); top = HAllocX(build_proc, sz, HEAP_XTRA); if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { - ASSERT(is_tuple_rel(term,base)); - *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, - &top, &MSO(build_proc), base); + ASSERT(is_tuple(term)); + *esp++ = copy_shallow(tuple_val(term), sz, &top, &MSO(build_proc)); } else { - *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), - base, NULL); + *esp++ = copy_struct(term, sz, &top, &MSO(build_proc)); } } else { @@ -2087,7 +2175,6 @@ restart: } break; case matchPushArrayAsList: - ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ tp = termp; ehp = HAllocX(build_proc, n*2, HEAP_XTRA); @@ -2103,7 +2190,6 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - ASSERT_HALFWORD(base == NULL); *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: @@ -2179,7 +2265,7 @@ restart: pc += n; break; case matchSelf: - *esp++ = c_p->common.id; + *esp++ = self->common.id; break; case matchWaste: --esp; @@ -2189,6 +2275,7 @@ restart: break; case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); + ASSERT(c_p == self); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); *esp++ = new_binary(build_proc, (byte *)dsbufp->str, dsbufp->str_len); @@ -2207,18 +2294,16 @@ restart: *return_flags |= MATCH_SET_EXCEPTION_TRACE; *esp++ = am_true; break; - case matchIsSeqTrace: - if (SEQ_TRACE_TOKEN(c_p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag -#endif - ) + case matchIsSeqTrace: + ASSERT(c_p == self); + if (have_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = am_true; else *esp++ = am_false; break; case matchSetSeqToken: - t = erts_seq_trace(c_p, esp[-1], esp[-2], 0); + ASSERT(c_p == self); + t = erts_seq_trace(c_p, esp[-1], esp[-2], 0); if (is_non_value(t)) { esp[-2] = FAIL_TERM; } else { @@ -2226,7 +2311,8 @@ restart: } --esp; break; - case matchSetSeqTokenFake: + case matchSetSeqTokenFake: + ASSERT(c_p == self); t = seq_trace_fake(c_p, esp[-1]); if (is_non_value(t)) { esp[-2] = FAIL_TERM; @@ -2235,12 +2321,9 @@ restart: } --esp; break; - case matchGetSeqToken: - if (SEQ_TRACE_TOKEN(c_p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag -#endif - ) + case matchGetSeqToken: + ASSERT(c_p == self); + if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); @@ -2263,49 +2346,62 @@ restart: ASSERT(is_immed(ehp[5])); } break; - case matchEnableTrace: + case matchEnableTrace: + ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - BEGIN_ATOMIC_TRACE(c_p); - set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), 0, n); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; } break; - case matchEnableTrace2: + case matchEnableTrace2: + ASSERT(c_p == self); n = erts_trace_flag2bit((--esp)[-1]); esp[-1] = FAIL_TERM; if (n) { - BEGIN_ATOMIC_TRACE(c_p); - if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { + if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) { /* Always take over the tracer of the current process */ - set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), 0, n); - esp[-1] = am_true; + set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n); + if (tmpp == c_p) + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + else + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + esp[-1] = am_true; } } break; - case matchDisableTrace: + case matchDisableTrace: + ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - BEGIN_ATOMIC_TRACE(c_p); - set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), n, 0); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; } break; - case matchDisableTrace2: + case matchDisableTrace2: + ASSERT(c_p == self); n = erts_trace_flag2bit((--esp)[-1]); esp[-1] = FAIL_TERM; if (n) { - BEGIN_ATOMIC_TRACE(c_p); - if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { + if ( (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, esp[0], ERTS_PROC_LOCKS_ALL))) { /* Always take over the tracer of the current process */ - set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), n, 0); - esp[-1] = am_true; + set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0); + if (tmpp == c_p) + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + else + erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + esp[-1] = am_true; } } break; - case matchCaller: + case matchCaller: + ASSERT(c_p == self); if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { @@ -2317,8 +2413,11 @@ restart: ehp[3] = make_small((Uint) cp[2]); } break; - case matchSilent: + case matchSilent: + ASSERT(c_p == self); --esp; + if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) + break; if (*esp == am_true) { erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT; @@ -2330,11 +2429,12 @@ restart: erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } break; - case matchTrace2: + case matchTrace2: + ASSERT(c_p == self); { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ - Eterm tracer = ERTS_TRACER_PROC(c_p); + ErtsTracer tracer = erts_tracer_nil; /* XXX Atomicity note: Not fully atomic. Default tracer * is sampled from current process but applied to * tracee and tracer later after releasing main @@ -2346,29 +2446,35 @@ restart: * {trace,[],[{{tracer,Tracer}}]} is much, much older. */ int cputs = 0; + erts_tracer_update(&tracer, ERTS_TRACER(c_p)); if (! erts_trace_flags(esp[-1], &d_flags, &tracer, &cputs) || ! erts_trace_flags(esp[-2], &e_flags, &tracer, &cputs) || cputs ) { (--esp)[-1] = FAIL_TERM; + ERTS_TRACER_CLEAR(&tracer); break; } erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); (--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer, d_flags, e_flags); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + ERTS_TRACER_CLEAR(&tracer); } break; - case matchTrace3: + case matchTrace3: + ASSERT(c_p == self); { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ - Eterm tracer = ERTS_TRACER_PROC(c_p); + ErtsTracer tracer = erts_tracer_nil; /* XXX Atomicity note. Not fully atomic. See above. * Above it could possibly be solved, but not here. */ int cputs = 0; Eterm tracee = (--esp)[0]; + + erts_tracer_update(&tracer, ERTS_TRACER(c_p)); if (! erts_trace_flags(esp[-1], &d_flags, &tracer, &cputs) || ! erts_trace_flags(esp[-2], &e_flags, &tracer, &cputs) || @@ -2376,6 +2482,7 @@ restart: ! (tmpp = get_proc(c_p, ERTS_PROC_LOCK_MAIN, tracee, ERTS_PROC_LOCKS_ALL))) { (--esp)[-1] = FAIL_TERM; + ERTS_TRACER_CLEAR(&tracer); break; } if (tmpp == c_p) { @@ -2389,6 +2496,7 @@ restart: erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } + ERTS_TRACER_CLEAR(&tracer); } break; case matchCatch: /* Match success, now build result */ @@ -2401,17 +2509,10 @@ restart: case matchHalt: goto success; default: - erl_exit(1, "Internal error: unexpected opcode in match program."); + erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program."); } } fail: -#if HALFWORD_HEAP - if (c_p_checkpoint.p) { - /* Dispose garbage built by guards on caller heap */ - heap_checkpoint_revert(&c_p_checkpoint); - c_p_checkpoint.p = NULL; - } -#endif *return_flags = 0U; if (fail_label >= 0) { /* We failed during a "TryMeElse", lets restart, with the next match @@ -2425,25 +2526,22 @@ success: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } #endif - esdp->current_process = current_scheduled; - - END_ATOMIC_TRACE(c_p); + if (esdp) + esdp->current_process = current_scheduled; return ret; #undef FAIL #undef FAIL_TERM -#undef BEGIN_ATOMIC_TRACE -#undef END_ATOMIC_TRACE } @@ -2574,21 +2672,14 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) { Eterm elem = handle->dbterm->tpl[position]; if (!is_header(elem)) { -#if HALFWORD_HEAP - if (!is_immed(elem) - && !handle->tb->common.compress - && !(handle->abs_vec && handle->abs_vec[position])) { - return rterm2wterm(elem, handle->dbterm->tpl); - } -#endif return elem; } ASSERT(((DbTableCommon*)handle->tb)->compress); - ASSERT(!handle->mustResize); + ASSERT(!(handle->flags & DB_MUST_RESIZE)); handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, handle->dbterm); - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; return handle->dbterm->tpl[position]; } @@ -2608,9 +2699,6 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; -#if HALFWORD_HEAP - Eterm* old_base; -#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; @@ -2621,21 +2709,14 @@ void db_do_update_element(DbUpdateHandle* handle, #endif return; } - if (!handle->mustResize) { + if (!(handle->flags & DB_MUST_RESIZE)) { if (handle->tb->common.compress) { handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, handle->dbterm); - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; oldval = handle->dbterm->tpl[position]; - #if HALFWORD_HEAP - old_base = NULL; - #endif } else { - #if HALFWORD_HEAP - ASSERT(!handle->abs_vec); - old_base = handle->dbterm->tpl; - #endif if (is_boxed(newval)) { newp = boxed_val(newval); switch (*newp & _TAG_HEADER_MASK) { @@ -2645,7 +2726,7 @@ void db_do_update_element(DbUpdateHandle* handle, case _TAG_HEADER_HEAP_BIN: newval_sz = header_arity(*newp) + 1; if (is_boxed(oldval)) { - oldp = boxed_val_rel(oldval,old_base); + oldp = boxed_val(oldval); switch (*oldp & _TAG_HEADER_MASK) { case _TAG_HEADER_POS_BIG: case _TAG_HEADER_NEG_BIG: @@ -2665,40 +2746,20 @@ void db_do_update_element(DbUpdateHandle* handle, } } } -#if HALFWORD_HEAP - else { - old_base = (handle->tb->common.compress - || (handle->abs_vec && handle->abs_vec[position])) ? - NULL : handle->dbterm->tpl; - } -#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); + oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; - handle->mustResize = 1; - -#if HALFWORD_HEAP - if (old_base && newval_sz > 0) { - ASSERT(!handle->tb->common.compress); - if (!handle->abs_vec) { - int i = header_arity(handle->dbterm->tpl[0]); - handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); - sys_memset(handle->abs_vec, 0, i+1); - /* abs_vec[0] not used */ - } - handle->abs_vec[position] = 1; - } -#endif + handle->flags |= DB_MUST_RESIZE; } static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, @@ -2797,7 +2858,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, tpl[arity + 1] = alloc_size; tmp_offheap.first = NULL; - tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); + tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); dest->first_oh = tmp_offheap.first; for (i=1; i<=arity; i++) { if (i != tb->keypos) { @@ -2816,7 +2877,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); dest->debug_clone = dbg_top; tmp_offheap.first = dest->first_oh; - copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); + copy_struct(obj, dest->size, &dbg_top, &tmp_offheap); dest->first_oh = tmp_offheap.first; ASSERT(dbg_top == dest->debug_clone + dest->size); } @@ -2863,7 +2924,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) newp->size = size; top = newp->tpl; tmp_offheap.first = NULL; - copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); + copy_struct(obj, size, &top, &tmp_offheap); newp->first_oh = tmp_offheap.first; #ifdef DEBUG_CLONE newp->debug_clone = NULL; @@ -2946,29 +3007,8 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) tmp_offheap.first = NULL; - #if HALFWORD_HEAP - if (handle->abs_vec) { - int i, arity = header_arity(handle->dbterm->tpl[0]); - - top[0] = tpl[0]; - top += arity + 1; - for (i=1; i<=arity; i++) { - Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; - - newDbTerm->tpl[i] = copy_struct_rel(tpl[i], - size_object_rel(tpl[i],src_base), - &top, &tmp_offheap, src_base, - newDbTerm->tpl); - } - newDbTerm->first_oh = tmp_offheap.first; - ASSERT((byte*)top <= (newp + alloc_sz)); - erts_free(ERTS_ALC_T_TMP, handle->abs_vec); - } - else - #endif /* HALFWORD_HEAP */ { - copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, - &tmp_offheap, tpl, top); + copy_struct(make_tuple(tpl), handle->new_size, &top, &tmp_offheap); newDbTerm->first_oh = tmp_offheap.first; ASSERT((byte*)top == (newp + alloc_sz)); } @@ -2980,27 +3020,34 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, { Eterm* hp = *hpp; int i, arity = arityval(bp->tpl[0]); + ErtsHeapFactory factory; hp[0] = bp->tpl[0]; *hpp += arity + 1; - hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], - size_object_rel(bp->tpl[tb->keypos], bp->tpl), - hpp, off_heap, bp->tpl, NULL); + hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos], + size_object(bp->tpl[tb->keypos]), + hpp, off_heap); + + erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); + for (i=arity; i>0; i--) { if (i != tb->keypos) { if (is_immed(bp->tpl[i])) { hp[i] = bp->tpl[i]; } else { - hp[i] = erts_decode_ext_ets(hpp, off_heap, + hp[i] = erts_decode_ext_ets(&factory, elem2ext(bp->tpl, i)); } } } + *hpp = factory.hp; + erts_factory_close(&factory); + ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE - ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); + ASSERT(EQ(make_tuple(hp),make_tuple(bp->debug_clone))); #endif return make_tuple(hp); } @@ -3016,21 +3063,22 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, if (tb->compress && pos != tb->keypos) { byte* ext = elem2ext(obj->tpl, pos); Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; - Eterm* hp = HAlloc(p, sz); - Eterm* endp = hp + sz; - Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); - *hpp = hp; - hp += extra; - HRelease(p, endp, hp); + Eterm copy; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, sz); + copy = erts_decode_ext_ets(&factory, ext); + *hpp = erts_produce_heap(&factory, extra, 0); + erts_factory_close(&factory); #ifdef DEBUG_CLONE - ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); + ASSERT(EQ(copy, obj->debug_clone[pos])); #endif return copy; } else { - Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); + Uint sz = size_object(obj->tpl[pos]); *hpp = HAlloc(p, sz + extra); - return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); + return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p)); } } @@ -3135,39 +3183,86 @@ int db_is_variable(Eterm obj) return N; } +/* check if node is (or contains) a map + * return 1 if node contains a map + * return 0 otherwise + */ + +int db_has_map(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + if (is_list(node)) { + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + } else if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } else if is_map(node) { + DESTROY_ESTACK(s); + return 1; + } + } + DESTROY_ESTACK(s); + return 0; +} /* check if obj is (or contains) a variable */ /* return 1 if obj contains a variable or underscore */ /* return 0 if obj is fully ground */ -int db_has_variable(Eterm obj) -{ - switch(obj & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: { - while (is_list(obj)) { - if (db_has_variable(CAR(list_val(obj)))) +int db_has_variable(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + switch(node & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + break; + case TAG_PRIMARY_BOXED: + if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } else if (is_flatmap(node)) { + Eterm *values = flatmap_get_values(flatmap_val(node)); + Uint size = flatmap_get_size(flatmap_val(node)); + ESTACK_PUSH(s, ((flatmap_t *) flatmap_val(node))->keys); + while (size--) { + ESTACK_PUSH(s, *(values++)); + } + } else if (is_map(node)) { /* other map-nodes or map-heads */ + Eterm *ptr = hashmap_val(node); + int i = hashmap_bitcount(MAP_HEADER_VAL(*ptr)); + ptr += MAP_HEADER_ARITY(*ptr); + while(i--) { ESTACK_PUSH(s, *++ptr); } + } + break; + case TAG_PRIMARY_IMMED1: + if (node == am_Underscore || db_is_variable(node) >= 0) { + DESTROY_ESTACK(s); return 1; - obj = CDR(list_val(obj)); - } - return(db_has_variable(obj)); /* Non wellformed list or [] */ - } - case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(obj)) { - return 0; - } else { - Eterm *tuple = tuple_val(obj); - int arity = arityval(*tuple++); - while(arity--) { - if (db_has_variable(*tuple)) - return 1; - tuple++; } - return(0); + break; } - case TAG_PRIMARY_IMMED1: - if (obj == am_Underscore || db_is_variable(obj) >= 0) - return 1; } + DESTROY_ESTACK(s); return 0; } @@ -3191,20 +3286,20 @@ int erts_db_is_compiled_ms(Eterm term) ** Utility to add an error */ -static void add_dmc_err(DMCErrInfo *err_info, - char *str, - int variable, - Eterm term, - DMCErrorSeverity severity) +static void vadd_dmc_err(DMCErrInfo *err_info, + DMCErrorSeverity severity, + int variable, + const char *str, + ...) { + DMCError *e; + va_list args; + va_start(args, str); + + /* Linked in in reverse order, to ease the formatting */ - DMCError *e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError)); - if (term != 0UL) { - erts_snprintf(e->error_string, DMC_ERR_STR_LEN, str, term); - } else { - strncpy(e->error_string, str, DMC_ERR_STR_LEN); - e->error_string[DMC_ERR_STR_LEN] ='\0'; - } + e = erts_alloc(ERTS_ALC_T_DB_DMC_ERROR, sizeof(DMCError)); + erts_vsnprintf(e->error_string, DMC_ERR_STR_LEN, str, args); e->variable = variable; e->severity = severity; e->next = err_info->first; @@ -3214,8 +3309,11 @@ static void add_dmc_err(DMCErrInfo *err_info, err_info->first = e; if (severity >= dmcError) err_info->error_added = 1; + + va_end(args); } + /* ** Handle one term in the match expression (not the guard) */ @@ -3227,11 +3325,9 @@ static DMCRet dmc_one_term(DMCContext *context, { Sint n; Eterm *hp; - ErlHeapFragment *tmp_mb; Uint sz, sz2, sz3; Uint i, j; - switch (c & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_IMMED1: if ((n = db_is_variable(c)) >= 0) { /* variable */ @@ -3318,30 +3414,24 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*text, n); DMC_PUSH(*stack, c); break; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): + if (is_flatmap(c)) + n = flatmap_get_size(flatmap_val(c)); + else + n = hashmap_size(c); + DMC_PUSH(*text, matchPushM); + ++(context->stack_used); + DMC_PUSH(*text, n); + DMC_PUSH(*stack, c); + break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): { Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(thing_arityval(ref_val[0]) == 3); - fiddle.t[0] = ref_val[0]; - fiddle.t[1] = ref_val[1]; - DMC_PUSH(*text, fiddle.u); - fiddle.t[0] = ref_val[2]; - fiddle.t[1] = ref_val[3]; - DMC_PUSH(*text, fiddle.u); - } -#else n = thing_arityval(ref_val[0]); for (i = 0; i <= n; ++i) { DMC_PUSH(*text, ref_val[i]); } -#endif break; } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): @@ -3350,77 +3440,51 @@ static DMCRet dmc_one_term(DMCContext *context, Eterm* bval = big_val(c); n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(n >= 1); - fiddle.t[0] = bval[0]; - fiddle.t[1] = bval[1]; - DMC_PUSH(*text, fiddle.u); - for (i = 2; i <= n; ++i) { - fiddle.t[0] = bval[i]; - if (++i <= n) { - fiddle.t[1] = bval[i]; - } else { - fiddle.t[1] = (Uint) 0; - } - DMC_PUSH(*text, fiddle.u); - } - } -#else for (i = 0; i <= n; ++i) { DMC_PUSH(*text, (Uint) bval[i]); } -#endif break; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - fiddle.t[0] = float_val(c)[1]; - fiddle.t[1] = float_val(c)[2]; - DMC_PUSH(*text, fiddle.u); - } -#else DMC_PUSH(*text, (Uint) float_val(c)[1]); #ifdef ARCH_64 DMC_PUSH(*text, (Uint) 0); #else DMC_PUSH(*text, (Uint) float_val(c)[2]); #endif -#endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ - /* - ** Make a private copy... - */ - n = size_object(c); - tmp_mb = new_message_buffer(n); - hp = tmp_mb->mem; DMC_PUSH(*text, matchEqBin); - DMC_PUSH(*text, copy_struct(c, n, &hp, &(tmp_mb->off_heap))); - tmp_mb->next = context->save; - context->save = tmp_mb; + DMC_PUSH(*text, dmc_private_copy(context, c)); break; } break; } default: - erl_exit(1, "db_match_compile: " + erts_exit(ERTS_ERROR_EXIT, "db_match_compile: " "Bad object on heap: 0x%bex\n", c); } return retOk; } /* +** Make a private copy of a term in a context. +*/ + +static Eterm +dmc_private_copy(DMCContext *context, Eterm c) +{ + Uint n = size_object(c); + ErlHeapFragment *tmp_mb = new_message_buffer(n); + Eterm *hp = tmp_mb->mem; + Eterm copy = copy_struct(c, n, &hp, &(tmp_mb->off_heap)); + tmp_mb->next = context->save; + context->save = tmp_mb; + return copy; +} + +/* ** Match guard compilation */ @@ -3448,24 +3512,21 @@ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, context->stack_need = context->stack_used; } -#define RETURN_ERROR_X(String, X, Y, ContextP, ConstantF) \ -do { \ -if ((ContextP)->err_info != NULL) { \ - (ConstantF) = 0; \ - add_dmc_err((ContextP)->err_info, String, X, Y, dmcError); \ - return retOk; \ -} else \ - return retFail; \ -} while(0) +#define RETURN_ERROR_X(VAR, ContextP, ConstantF, String, ARG) \ + (((ContextP)->err_info != NULL) \ + ? ((ConstantF) = 0, \ + vadd_dmc_err((ContextP)->err_info, dmcError, VAR, String, ARG), \ + retOk) \ + : retFail) #define RETURN_ERROR(String, ContextP, ConstantF) \ - RETURN_ERROR_X(String, -1, 0UL, ContextP, ConstantF) + return RETURN_ERROR_X(-1, ContextP, ConstantF, String, 0) #define RETURN_VAR_ERROR(String, N, ContextP, ConstantF) \ - RETURN_ERROR_X(String, N, 0UL, ContextP, ConstantF) + return RETURN_ERROR_X(N, ContextP, ConstantF, String, 0) #define RETURN_TERM_ERROR(String, T, ContextP, ConstantF) \ - RETURN_ERROR_X(String, -1, T, ContextP, ConstantF) + return RETURN_ERROR_X(-1, ContextP, ConstantF, String, T) #define WARNING(String, ContextP) \ add_dmc_err((ContextP)->err_info, String, -1, 0UL, dmcWarning) @@ -3511,57 +3572,78 @@ static DMCRet dmc_list(DMCContext *context, return retOk; } -static DMCRet dmc_tuple(DMCContext *context, - DMCHeap *heap, - DMC_STACK_TYPE(UWord) *text, - Eterm t, - int *constant) +static void +dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text, + int textpos, Eterm *p, Uint nelems) { DMC_STACK_TYPE(UWord) instr_save; + Uint i; + + DMC_INIT_STACK(instr_save); + while (DMC_STACK_NUM(*text) > textpos) { + DMC_PUSH(instr_save, DMC_POP(*text)); + } + for (i = nelems; i--;) { + do_emit_constant(context, text, p[i]); + } + while(!DMC_EMPTY(instr_save)) { + DMC_PUSH(*text, DMC_POP(instr_save)); + } + DMC_FREE(instr_save); +} + +static DMCRet +dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm *p, Uint nelems, int *constant) +{ int all_constant = 1; int textpos = DMC_STACK_NUM(*text); - Eterm *p = tuple_val(t); - Uint nelems = arityval(*p); Uint i; - int c; - DMCRet ret; /* - ** We remember where we started to layout code, + ** We remember where we started to layout code, ** assume all is constant and back up and restart if not so. - ** The tuple should be laid out with the last element first, - ** so we can memcpy the tuple to the eheap. + ** The array should be laid out with the last element first, + ** so we can memcpy it to the eheap. */ - for (i = nelems; i > 0; --i) { - if ((ret = dmc_expr(context, heap, text, p[i], &c)) != retOk) - return ret; - if (!c && all_constant) { - all_constant = 0; - if (i < nelems) { - Uint j; + for (i = nelems; i--;) { + DMCRet ret; + int c; - /* - * Oops, we need to relayout the constants. - * Save the already laid out instructions. - */ - DMC_INIT_STACK(instr_save); - while (DMC_STACK_NUM(*text) > textpos) - DMC_PUSH(instr_save, DMC_POP(*text)); - for (j = nelems; j > i; --j) - do_emit_constant(context, text, p[j]); - while(!DMC_EMPTY(instr_save)) - DMC_PUSH(*text, DMC_POP(instr_save)); - DMC_FREE(instr_save); - } - } else if (c && !all_constant) { - /* push a constant */ - do_emit_constant(context, text, p[i]); - } + ret = dmc_expr(context, heap, text, p[i], &c); + if (ret != retOk) { + return ret; + } + if (!c && all_constant) { + all_constant = 0; + if (i < nelems - 1) { + dmc_rearrange_constants(context, text, textpos, + p + i + 1, nelems - i - 1); + } + } else if (c && !all_constant) { + do_emit_constant(context, text, p[i]); + } + } + *constant = all_constant; + return retOk; +} + +static DMCRet +dmc_tuple(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant) +{ + int all_constant; + Eterm *p = tuple_val(t); + Uint nelems = arityval(*p); + DMCRet ret; + + ret = dmc_array(context, heap, text, p + 1, nelems, &all_constant); + if (ret != retOk) { + return ret; } - if (all_constant) { - *constant = 1; - return retOk; + *constant = 1; + return retOk; } DMC_PUSH(*text, matchMkTuple); DMC_PUSH(*text, nelems); @@ -3570,6 +3652,93 @@ static DMCRet dmc_tuple(DMCContext *context, return retOk; } +static DMCRet +dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant) +{ + int nelems; + int constant_values; + DMCRet ret; + if (is_flatmap(t)) { + flatmap_t *m = (flatmap_t *)flatmap_val(t); + Eterm *values = flatmap_get_values(m); + + nelems = flatmap_get_size(m); + ret = dmc_array(context, heap, text, values, nelems, &constant_values); + + if (ret != retOk) { + return ret; + } + if (constant_values) { + *constant = 1; + return retOk; + } + DMC_PUSH(*text, matchPushC); + DMC_PUSH(*text, dmc_private_copy(context, m->keys)); + if (++context->stack_used > context->stack_need) { + context->stack_need = context->stack_used; + } + DMC_PUSH(*text, matchMkFlatMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + *constant = 0; + return retOk; + } else { + DECLARE_WSTACK(wstack); + Eterm *kv; + int c; + + ASSERT(is_hashmap(t)); + + hashmap_iterator_init(&wstack, t, 1); + constant_values = 1; + nelems = hashmap_size(t); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (!c) + constant_values = 0; + } + + if (constant_values) { + *constant = 1; + DESTROY_WSTACK(wstack); + return retOk; + } + + *constant = 0; + + hashmap_iterator_init(&wstack, t, 1); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + /* push key */ + if ((ret = dmc_expr(context, heap, text, CAR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CAR(kv)); + } + /* push value */ + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CDR(kv)); + } + } + DMC_PUSH(*text, matchMkHashMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + DESTROY_WSTACK(wstack); + return retOk; + } +} + static DMCRet dmc_whole_expression(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -3604,24 +3773,8 @@ static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, MatchOps instr = matchPushV; ASSERT(n < heap->vars_used && v->is_bound); - if (context->is_guard) { - #if HALFWORD_HEAP - if (!v->first_guard_label) { - v->first_guard_label = DMC_STACK_NUM(*text); - ASSERT(v->first_guard_label); - instr = matchPushVGuard; /* may be changed to PushVResult below */ - } - #endif - } - else { /* body */ - #if HALFWORD_HEAP - if (v->first_guard_label) { - /* Avoid double-copy, copy to result heap at first encounter in guard */ - DMC_POKE(*text, v->first_guard_label, matchPushVResult); - v->is_in_body = 1; - } - #endif - if (!v->is_in_body) { + if (!context->is_guard) { + if(!v->is_in_body) { instr = matchPushVResult; v->is_in_body = 1; } @@ -3639,7 +3792,7 @@ static DMCRet dmc_variable(DMCContext *context, Uint n = db_is_variable(t); if (n >= heap->vars_used || !heap->vars[n].is_bound) { - RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); + RETURN_VAR_ERROR("Variable $%%d is unbound.", n, context, *constant); } dmc_add_pushv_variant(context, heap, text, n); @@ -3971,7 +4124,30 @@ static DMCRet dmc_exception_trace(DMCContext *context, return retOk; } - +static int check_trace(const char* op, + DMCContext *context, + int *constant, + int need_cflags, + int allow_in_guard, + DMCRet* retp) +{ + if (!(context->cflags & DCOMP_TRACE)) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "used in wrong dialect.", op); + return 0; + } + if ((context->cflags & need_cflags) != need_cflags) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "not allow for this trace event.", op); + return 0; + } + if (context->is_guard && !allow_in_guard) { + *retp = RETURN_ERROR_X(-1, context, *constant, "Special form '%s' " + "called in guard context.", op); + return 0; + } + return 1; +} static DMCRet dmc_is_seq_trace(DMCContext *context, DMCHeap *heap, @@ -3981,12 +4157,11 @@ static DMCRet dmc_is_seq_trace(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'is_seq_trace' used in wrong dialect.", - context, - *constant); - } + if (!check_trace("is_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 1, &ret)) + return ret; + if (a != 1) { RETURN_TERM_ERROR("Special form 'is_seq_trace' called with " "arguments in %T.", t, context, *constant); @@ -4010,16 +4185,8 @@ static DMCRet dmc_set_seq_token(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'set_seq_token' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'set_seq_token' called in " - "guard context.", context, *constant); - } + if (!check_trace("set_seq_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 3) { RETURN_TERM_ERROR("Special form 'set_seq_token' called with wrong " @@ -4056,16 +4223,11 @@ static DMCRet dmc_get_seq_token(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; + + if (!check_trace("get_seq_token", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'get_seq_token' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'get_seq_token' called in " - "guard context.", context, *constant); - } if (a != 1) { RETURN_TERM_ERROR("Special form 'get_seq_token' called with " "arguments in %T.", t, context, @@ -4129,16 +4291,10 @@ static DMCRet dmc_process_dump(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'process_dump' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'process_dump' called in " - "guard context.", context, *constant); - } + DMCRet ret; + + if (!check_trace("process_dump", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 1) { RETURN_TERM_ERROR("Special form 'process_dump' called with " @@ -4162,17 +4318,8 @@ static DMCRet dmc_enable_trace(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'enable_trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'enable_trace' called in guard context.", - context, - *constant); - } + if (!check_trace("enable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 2: @@ -4221,18 +4368,9 @@ static DMCRet dmc_disable_trace(DMCContext *context, Uint a = arityval(*p); DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'disable_trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'disable_trace' called in guard context.", - context, - *constant); - } + if (!check_trace("disable_trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 2: @@ -4282,17 +4420,8 @@ static DMCRet dmc_trace(DMCContext *context, DMCRet ret; int c; - - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'trace' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'trace' called in guard context.", - context, - *constant); - } + if (!check_trace("trace", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; switch (a) { case 3: @@ -4353,16 +4482,11 @@ static DMCRet dmc_caller(DMCContext *context, { Eterm *p = tuple_val(t); Uint a = arityval(*p); + DMCRet ret; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'caller' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'caller' called in " - "guard context.", context, *constant); - } + if (!check_trace("caller", context, constant, + (DCOMP_CALL_TRACE|DCOMP_ALLOW_TRACE_OPS), 0, &ret)) + return ret; if (a != 1) { RETURN_TERM_ERROR("Special form 'caller' called with " @@ -4388,15 +4512,8 @@ static DMCRet dmc_silent(DMCContext *context, DMCRet ret; int c; - if (!(context->cflags & DCOMP_TRACE)) { - RETURN_ERROR("Special form 'silent' used in wrong dialect.", - context, - *constant); - } - if (context->is_guard) { - RETURN_ERROR("Special form 'silent' called in " - "guard context.", context, *constant); - } + if (!check_trace("silent", context, constant, DCOMP_ALLOW_TRACE_OPS, 0, &ret)) + return ret; if (a != 2) { RETURN_TERM_ERROR("Special form 'silent' called with wrong " @@ -4537,7 +4654,7 @@ static DMCRet dmc_fun(DMCContext *context, DMC_PUSH(*text, matchCall3); break; default: - erl_exit(1,"ets:match() internal error, " + erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, " "guard with more than 3 arguments."); } DMC_PUSH(*text, (UWord) b->biff); @@ -4564,7 +4681,10 @@ static DMCRet dmc_expr(DMCContext *context, return ret; break; case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(t)) { + if (is_map(t)) { + return dmc_map(context, heap, text, t, constant); + } + if (!is_tuple(t)) { goto simple_term; } p = tuple_val(t); @@ -4814,7 +4934,7 @@ static Uint my_size_object(Eterm t) tmp == am_const) { sum += size_object(tuple_val(t)[2]); } else { - erl_exit(1,"Internal error, sizing unrecognized object in " + erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in " "(d)ets:match compilation."); } break; @@ -4839,7 +4959,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) *hp += 2; break; case TAG_PRIMARY_BOXED: - if (BOXED_IS_TUPLE(t)) { + if (is_tuple(t)) { if (arityval(*tuple_val(t)) == 1 && is_tuple(a = tuple_val(t)[1])) { Uint i,n; @@ -4859,7 +4979,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) sz = size_object(b); ret = copy_struct(b,sz,hp,off_heap); } else { - erl_exit(1, "Trying to constant-copy non constant expression " + erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression " "0x%bex in (d)ets:match compilation.", t); } } else { @@ -4933,13 +5053,16 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) return THE_NON_VALUE; } if (trace) { - lint_res = db_match_set_lint(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE); - mps = db_match_set_compile(p, spec, DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE); + const Uint cflags = (DCOMP_TRACE | DCOMP_FAKE_DESTRUCTIVE | + DCOMP_CALL_TRACE | DCOMP_ALLOW_TRACE_OPS); + lint_res = db_match_set_lint(p, spec, cflags); + mps = db_match_set_compile(p, spec, cflags); } else { - lint_res = db_match_set_lint(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); - mps = db_match_set_compile(p, spec, DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); + const Uint cflags = (DCOMP_TABLE | DCOMP_FAKE_DESTRUCTIVE); + lint_res = db_match_set_lint(p, spec, cflags); + mps = db_match_set_compile(p, spec, cflags); } - + if (mps == NULL) { hp = HAlloc(p,3); ret = TUPLE2(hp, am_error, lint_res); @@ -4970,8 +5093,10 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) } save_cp = p->cp; p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, - ERTS_PAM_COPY_RESULT, &ret_flags); + res = erts_match_set_run_trace(p, p, + mps, arr, n, + ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, + &ret_flags); p->cp = save_cp; } else { n = 0; @@ -5046,16 +5171,14 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra) { Uint32 dummy; - Eterm* base; Eterm res; if (tb->compress) { obj = db_alloc_tmp_uncompressed(tb, obj); - base = NULL; } - else base = HALFWORD_HEAP ? obj->tpl : NULL; - res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + res = db_prog_match(c_p, c_p, + bprog, make_tuple(obj->tpl), NULL, 0, ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { @@ -5109,6 +5232,18 @@ void db_match_dis(Binary *bp) ++t; erts_printf("Tuple\t%beu\n", n); break; + case matchMap: + ++t; + n = *t; + ++t; + erts_printf("Map\t%beu\n", n); + break; + case matchKey: + ++t; + p = (Eterm) *t; + ++t; + erts_printf("Key\t%p (%T)\n", t, p); + break; case matchPushT: ++t; n = *t; @@ -5119,10 +5254,20 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushL\n"); break; + case matchPushM: + ++t; + n = *t; + ++t; + erts_printf("PushM\t%beu\n", n); + break; case matchPop: ++t; erts_printf("Pop\n"); break; + case matchSwap: + ++t; + erts_printf("Swap\n"); + break; case matchBind: ++t; n = *t; @@ -5154,7 +5299,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", rt->data.ui[ri]); #else erts_printf("0x%08bex", rt->data.ui[ri]); @@ -5178,7 +5323,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", *et); #else erts_printf("0x%08bex", *et); @@ -5235,6 +5380,18 @@ void db_match_dis(Binary *bp) ++t; erts_printf("MkTuple\t%beu\n", n); break; + case matchMkFlatMap: + ++t; + n = *t; + ++t; + erts_printf("MkFlatMap\t%beu\n", n); + break; + case matchMkHashMap: + ++t; + n = *t; + ++t; + erts_printf("MkHashMap\t%beu\n", n); + break; case matchOr: ++t; n = *t; @@ -5289,13 +5446,6 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushV\t%beu\n", n); break; - #if HALFWORD_HEAP - case matchPushVGuard: - n = (Uint) *++t; - ++t; - erts_printf("PushVGuard\t%beu\n", n); - break; - #endif case matchPushVResult: n = (Uint) *++t; ++t; |