diff options
Diffstat (limited to 'erts/emulator/beam/erl_db_util.c')
| -rw-r--r-- | erts/emulator/beam/erl_db_util.c | 773 |
1 files changed, 522 insertions, 251 deletions
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 2852fb93fe..d3e31da413 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -57,6 +57,7 @@ DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY +#define HEAP_XTRA 100 /* ** Some convenience macros for stacks (DMC == db_match_compile) @@ -229,6 +230,11 @@ typedef enum { matchCall2, matchCall3, matchPushV, +#if HALFWORD_HEAP + matchPushVGuard, /* First guard-only variable reference */ +#endif + matchPushVResult, /* First variable reference in result, or (if HALFWORD) + in guard if also referenced in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -292,11 +298,19 @@ DMC_DECLARE_STACK_TYPE(unsigned); ** Data about the heap during compilation */ +typedef struct DMCVariable { + int is_bound; + int is_in_body; +#if HALFWORD_HEAP + int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ +#endif +} DMCVariable; + typedef struct DMCHeap { int size; - unsigned def[DMC_DEFAULT_SIZE]; - unsigned *data; - int used; + DMCVariable vars_def[DMC_DEFAULT_SIZE]; + DMCVariable* vars; + int vars_used; } DMCHeap; /* @@ -323,7 +337,6 @@ typedef struct dmc_context { Eterm *bodyexpr; int num_match; int current_match; - int eheap_need; Uint cflags; int is_guard; /* 1 if in guard, 0 if in body */ int special; /* 1 if the head in the match was a single expression */ @@ -346,9 +359,22 @@ typedef struct dmc_context { #define ERTS_DEFAULT_MS_HEAP_SIZE 128 +/* Runtime info about a $-variable +*/ +typedef struct MatchVariable { + Eterm term; +#ifdef DEBUG + Process* proc; + Eterm* base; +#endif +} MatchVariable; + typedef struct { Process process; - Eterm *heap; + union { + Eterm* heap; + MatchVariable* variables; /* first on "heap" */ + }u; Eterm default_heap[ERTS_DEFAULT_MS_HEAP_SIZE]; } ErtsMatchPseudoProcess; @@ -371,10 +397,10 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) } #endif if (!keep_heap) { - if (mpsp->heap != &mpsp->default_heap[0]) { + if (mpsp->u.heap != mpsp->default_heap) { /* Have to be done *after* call to erts_cleanup_empty_process() */ - erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->heap); - mpsp->heap = &mpsp->default_heap[0]; + erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->u.heap); + mpsp->u.heap = mpsp->default_heap; } #ifdef DEBUG else { @@ -398,7 +424,7 @@ create_match_pseudo_process(void) mpsp = (ErtsMatchPseudoProcess *)erts_alloc(ERTS_ALC_T_DB_MS_PSDO_PROC, sizeof(ErtsMatchPseudoProcess)); erts_init_empty_process(&mpsp->process); - mpsp->heap = &mpsp->default_heap[0]; + mpsp->u.heap = mpsp->default_heap; return mpsp; } @@ -422,11 +448,11 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) mpsp = match_pseudo_process; cleanup_match_pseudo_process(mpsp, 0); #endif - if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE) - mpsp->heap = (Eterm *) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, - heap_size*sizeof(Uint)); + if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) { + mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size); + } else { - ASSERT(mpsp->heap == &mpsp->default_heap[0]); + ASSERT(mpsp->u.heap == mpsp->default_heap); } return mpsp; } @@ -467,23 +493,6 @@ erts_match_set_release_result(Process* c_p) static erts_smp_atomic_t trace_control_word; - -Eterm -erts_ets_copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; - - res = copy_struct(obj, size, &hp, &MSO(to)); -#ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(1, "copy not equal to source\n"); - } -#endif - return res; -} - /* This needs to be here, before the bif table... */ static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val); @@ -869,9 +878,9 @@ static DMCRet dmc_one_term(DMCContext *context, #ifdef DMC_DEBUG static int test_disassemble_next = 0; -static void db_match_dis(Binary *prog); +void db_match_dis(Binary *prog); #define TRACE erts_fprintf(stderr,"Trace: %s:%d\n",__FILE__,__LINE__) -#define FENCE_PATTERN_SIZE 1 +#define FENCE_PATTERN_SIZE (1*sizeof(Uint)) #define FENCE_PATTERN 0xDEADBEEFUL #else #define TRACE /* Nothing */ @@ -1179,14 +1188,14 @@ done: } Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, Uint32 *return_flags) { Eterm ret; - ret = db_prog_match(p, mpsp, - NIL, args, - num_args, return_flags); + ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + in_flags, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { erts_fprintf(stderr, "Failed\n"); @@ -1210,9 +1219,9 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, - args, NULL, - num_args, return_flags); + ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ERTS_PAM_CONTIGUOUS_TUPLE | ERTS_PAM_COPY_RESULT, + return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { erts_fprintf(stderr, "Failed\n"); @@ -1280,7 +1289,6 @@ Binary *db_match_compile(Eterm *matchexpr, int structure_checked; DMCRet res; int current_try_label; - Uint max_eheap_need; Binary *bp = NULL; unsigned clause_start; @@ -1293,27 +1301,24 @@ Binary *db_match_compile(Eterm *matchexpr, context.matchexpr = matchexpr; context.guardexpr = guards; context.bodyexpr = body; - context.eheap_need = 0; context.err_info = err_info; context.cflags = flags; heap.size = DMC_DEFAULT_SIZE; - heap.data = heap.def; + heap.vars = heap.vars_def; /* ** Compile the match expression */ restart: - heap.used = 0; - max_eheap_need = 0; + heap.vars_used = 0; for (context.current_match = 0; context.current_match < num_progs; ++context.current_match) { /* This loop is long, too long */ - memset(heap.data, 0, heap.size * sizeof(*heap.data)); + memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); t = context.matchexpr[context.current_match]; context.stack_used = 0; - context.eheap_need = 0; structure_checked = 0; if (context.current_match < num_progs - 1) { DMC_PUSH(text,matchTryMeElse); @@ -1485,10 +1490,6 @@ restart: if (current_try_label >= 0) { DMC_POKE(text, current_try_label, DMC_STACK_NUM(text)); } - /* So, how much eheap did this part of the match program need? */ - if (context.eheap_need > max_eheap_need) { - max_eheap_need = context.eheap_need; - } } /* for (context.current_match = 0 ...) */ @@ -1524,16 +1525,13 @@ restart: ret->saved_program_buf = NULL; ret->saved_program = NIL; ret->term_save = context.save; - ret->num_bindings = heap.used; + ret->num_bindings = heap.vars_used; ret->single_variable = context.special; sys_memcpy(ret->text, DMC_STACK_DATA(text), DMC_STACK_NUM(text) * sizeof(UWord)); - ret->heap_size = ((heap.used * sizeof(Eterm)) + - (max_eheap_need * sizeof(Eterm)) + - (context.stack_need * sizeof(Eterm *)) + - (3 * (FENCE_PATTERN_SIZE * sizeof(Eterm *)))); - ret->eheap_offset = heap.used + FENCE_PATTERN_SIZE; - ret->stack_offset = ret->eheap_offset + max_eheap_need + FENCE_PATTERN_SIZE; + ret->stack_offset = heap.vars_used*sizeof(MatchVariable) + FENCE_PATTERN_SIZE; + ret->heap_size = ret->stack_offset + context.stack_need * sizeof(Eterm*) + FENCE_PATTERN_SIZE; + #ifdef DMC_DEBUG ret->prog_end = ret->text + DMC_STACK_NUM(text); #endif @@ -1551,8 +1549,8 @@ error: /* Here is were we land when compilation failed. */ DMC_FREE(text); if (context.copy != NULL) free_message_buffer(context.copy); - if (heap.data != heap.def) - erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.data); + if (heap.vars != heap.vars_def) + erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.vars); return bp; } @@ -1597,7 +1595,7 @@ erts_match_prog_foreach_offheap(Binary *bprog, */ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) { - Eterm *hp = HAlloc(psp, arity * 2); + Eterm *hp = HAllocX(psp, arity * 2, HEAP_XTRA); Eterm ret = NIL; while (--arity >= 0) { ret = CONS(hp, arr[arity], ret); @@ -1606,15 +1604,82 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) return ret; } + +#if HALFWORD_HEAP +struct heap_checkpoint_t +{ + Process *p; + Eterm* htop; + ErlHeapFragment* mbuf; + unsigned used_size; + ErlOffHeap off_heap; +}; + +static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) +{ + hcp->p = p; + hcp->htop = HEAP_TOP(p); + hcp->mbuf = MBUF(p); + hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; + hcp->off_heap = MSO(p); +} + +static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) +{ + struct erl_off_heap_header* oh = MSO(hcp->p).first; + + if (oh != hcp->off_heap.first) { + ASSERT(oh != NULL); + if (hcp->off_heap.first) { + while (oh->next != hcp->off_heap.first) { + oh = oh->next; + } + oh->next = NULL; + } + erts_cleanup_offheap(&MSO(hcp->p)); + MSO(hcp->p) = hcp->off_heap; + } + if (MBUF(hcp->p) != hcp->mbuf) { + ErlHeapFragment* hf = MBUF(hcp->p); + ASSERT(hf != NULL); + if (hcp->mbuf) { + while (hf->next != hcp->mbuf) { + hf = hf->next; + } + hf->next = NULL; + } + free_message_buffer(MBUF(hcp->p)); + MBUF(hcp->p) = hcp->mbuf; + } + if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { + hcp->mbuf->used_size = hcp->used_size; + } + HEAP_TOP(hcp->p) = hcp->htop; +} +#endif /* HALFWORD_HEAP */ + +static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) +{ + if (!is_immed(term)) { + Uint sz = size_object_rel(term, base); + Eterm* top = HAllocX(p, sz, HEAP_XTRA); + return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); + } + return term; +} + + /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. -** the para meter 'arity' is only used if 'term' is actually an array, +** the parameter 'arity' is only used if 'term' is actually an array, ** i.e. 'DCOMP_TRACE' was specified */ -Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, +Eterm db_prog_match(Process *c_p, Binary *bprog, + Eterm term, Eterm* base, Eterm *termp, int arity, + enum erts_pam_run_flags in_flags, Uint32 *return_flags) { MatchProg *prog = Binary2MatchProg(bprog); @@ -1623,7 +1688,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, Eterm t; Eterm **sp; Eterm *esp; - Eterm *hp; + MatchVariable* variables; BeamInstr *cp; UWord *pc = prog->text; Eterm *ehp; @@ -1633,19 +1698,24 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, unsigned do_catch; ErtsMatchPseudoProcess *mpsp; Process *psp; + Process* build_proc; Process *tmpp; Process *current_scheduled; ErtsSchedulerData *esdp; Eterm (*bif)(Process*, ...); int fail_label; int atomic_trace; +#if HALFWORD_HEAP + struct heap_checkpoint_t c_p_checkpoint = {}; +#endif #ifdef DMC_DEBUG Uint *heap_fence; - Uint *eheap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ + ASSERT(base==NULL || HALFWORD_HEAP); + mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1655,7 +1725,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p); ASSERT(esdp != NULL); current_scheduled = esdp->current_process; - esdp->current_process = psp; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ atomic_trace = 0; @@ -1678,11 +1747,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, #ifdef DMC_DEBUG save_op = 0; - heap_fence = (Uint *) mpsp->heap + prog->eheap_offset - 1; - eheap_fence = (Uint *) mpsp->heap + prog->stack_offset - 1; - stack_fence = (Uint *) mpsp->heap + prog->heap_size - 1; + heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1; + stack_fence = (Eterm*)((char*) mpsp->u.heap + prog->heap_size) - 1; *heap_fence = FENCE_PATTERN; - *eheap_fence = FENCE_PATTERN; *stack_fence = FENCE_PATTERN; #endif /* DMC_DEBUG */ @@ -1696,36 +1763,48 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, *return_flags = 0U; + variables = mpsp->u.variables; +#if HALFWORD_HEAP + c_p_checkpoint.p = NULL; +#endif + restart: ep = &term; - esp = mpsp->heap + prog->stack_offset; + esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset); sp = (Eterm **) esp; - hp = mpsp->heap; - ehp = mpsp->heap + prog->eheap_offset; ret = am_true; do_catch = 0; fail_label = -1; + build_proc = psp; + esdp->current_process = psp; + ASSERT_HALFWORD(!c_p_checkpoint.p); + +#ifdef DEBUG + ASSERT(variables == mpsp->u.variables); + for (i=0; i<prog->num_bindings; i++) { + variables[i].term = THE_NON_VALUE; + variables[i].proc = NULL; + variables[i].base = base; + } +#endif for (;;) { -#ifdef DMC_DEBUG + + #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { erl_exit(1, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } - if (*eheap_fence != FENCE_PATTERN) { - erl_exit(1, "Eheap fence overwritten in db_prog_match after op " - "0x%08x, overwritten with 0x%08x.", save_op, - *eheap_fence); - } if (*stack_fence != FENCE_PATTERN) { erl_exit(1, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } save_op = *pc; -#endif + #endif switch (*pc++) { case matchTryMeElse: + ASSERT(fail_label == -1); fail_label = *pc++; break; case matchArray: /* only when DCOMP_TRACE, is always first @@ -1736,13 +1815,14 @@ restart: ep = termp; break; case matchArrayBind: /* When the array size is unknown. */ + ASSERT(termp); n = *pc++; - hp[n] = dpm_array_to_list(psp, termp, arity); + variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple(*ep)) + if (!is_tuple_rel(*ep,base)) FAIL(); - ep = tuple_val(*ep); + ep = tuple_val_rel(*ep,base); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1750,9 +1830,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple(*ep)) + if (!is_tuple_rel(*ep,base)) FAIL(); - tp = tuple_val(*ep); + tp = tuple_val_rel(*ep,base); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1762,12 +1842,12 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val(*ep); + ep = list_val_rel(*ep,base); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val(*ep); + *sp++ = list_val_rel(*ep,base); ++ep; break; case matchPop: @@ -1775,41 +1855,44 @@ restart: break; case matchBind: n = *pc++; - hp[n] = *ep++; + variables[n].term = *ep++; break; case matchCmp: n = *pc++; - if (!eq(hp[n],*ep)) + if (!eq_rel(variables[n].term, base, *ep, base)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq(*ep,t)) + if (!eq_rel(t,NULL,*ep,base)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float(*ep)) + if (!is_float_rel(*ep,base)) FAIL(); - if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) + if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; break; - case matchEqRef: - if (!is_ref(*ep)) + case matchEqRef: { + Eterm* epc = (Eterm*)pc; + if (!is_ref_rel(*ep,base)) FAIL(); - if (!eq(*ep, make_internal_ref((Uint *) pc))) + if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { FAIL(); - i = thing_arityval(*((Uint *) pc)); + } + i = thing_arityval(*epc); pc += TermWords(i+1); ++ep; break; + } case matchEqBig: - if (!is_big(*ep)) + if (!is_big_rel(*ep,base)) FAIL(); - tp = big_val(*ep); + tp = big_val_rel(*ep,base); { Eterm *epc = (Eterm *) pc; if (*tp != *epc) @@ -1825,7 +1908,8 @@ restart: ++ep; break; case matchEq: - t = (Eterm) *pc++; + t = (Eterm) *pc++; + ASSERT(is_immed(t)); if (t != *ep++) FAIL(); break; @@ -1833,25 +1917,32 @@ restart: ++ep; break; /* - * Here comes guard instructions + * Here comes guard & body instructions */ case matchPushC: /* Push constant */ - *esp++ = *pc++; + if ((in_flags & ERTS_PAM_COPY_RESULT) + && do_catch && !is_immed(*pc)) { + *esp++ = copy_object(*pc++, c_p); + } + else { + *esp++ = *pc++; + } break; case matchConsA: - ehp[1] = *--esp; - ehp[0] = esp[-1]; + ehp = HAllocX(build_proc, 2, HEAP_XTRA); + CDR(ehp) = *--esp; + CAR(ehp) = esp[-1]; esp[-1] = make_list(ehp); - ehp += 2; break; case matchConsB: - ehp[0] = *--esp; - ehp[1] = esp[-1]; + ehp = HAllocX(build_proc, 2, HEAP_XTRA); + CAR(ehp) = *--esp; + CDR(ehp) = esp[-1]; esp[-1] = make_list(ehp); - ehp += 2; break; case matchMkTuple: n = *pc++; + ehp = HAllocX(build_proc, n+1, HEAP_XTRA); t = make_tuple(ehp); *ehp++ = make_arityval(n); while (n--) { @@ -1861,7 +1952,7 @@ restart: break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp); + t = (*bif)(build_proc); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1872,7 +1963,7 @@ restart: break; case matchCall1: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1]); + t = (*bif)(build_proc, esp[-1]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1883,7 +1974,7 @@ restart: break; case matchCall2: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1], esp[-2]); + t = (*bif)(build_proc, esp[-1], esp[-2]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1895,7 +1986,7 @@ restart: break; case matchCall3: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1], esp[-2], esp[-3]); + t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1905,15 +1996,73 @@ restart: esp -= 2; esp[-1] = t; break; + + #if HALFWORD_HEAP + case matchPushVGuard: + if (!base) goto case_matchPushV; + /* Build NULL-based copy on pseudo heap for easy disposal */ + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].proc); + variables[n].term = copy_object_rel(psp, variables[n].term, base); + *esp++ = variables[n].term; + #ifdef DEBUG + variables[n].proc = psp; + variables[n].base = NULL; + #endif + break; + #endif + case matchPushVResult: + if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; + + /* Build (NULL-based) copy on callers heap */ + #if HALFWORD_HEAP + if (!do_catch && !c_p_checkpoint.p) { + heap_checkpoint_init(c_p, &c_p_checkpoint); + } + #endif + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].proc); + variables[n].term = copy_object_rel(c_p, variables[n].term, base); + *esp++ = variables[n].term; + #ifdef DEBUG + variables[n].proc = c_p; + variables[n].base = NULL; + #endif + break; case matchPushV: - *esp++ = hp[*pc++]; + case_matchPushV: + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].base); + *esp++ = variables[n].term; break; case matchPushExpr: - *esp++ = term; + if (in_flags & ERTS_PAM_COPY_RESULT) { + Uint sz; + Eterm* top; + sz = size_object_rel(term, base); + top = HAllocX(build_proc, sz, HEAP_XTRA); + if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { + ASSERT(is_tuple_rel(term,base)); + *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, + &top, &MSO(build_proc), base); + } + else { + *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), + base, NULL); + } + } + else { + *esp = term; + } break; case matchPushArrayAsList: + ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ tp = termp; + ehp = HAllocX(build_proc, n*2, HEAP_XTRA); *esp++ = make_list(ehp); while (n--) { *ehp++ = *tp++; @@ -1926,7 +2075,8 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - *esp++ = dpm_array_to_list(psp, termp, arity); + ASSERT_HALFWORD(base == NULL); + *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: if (*--esp != am_true) @@ -2012,7 +2162,7 @@ restart: case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); - *esp++ = new_binary(psp, (byte *)dsbufp->str, (int)dsbufp->str_len); + *esp++ = new_binary(build_proc, (byte *)dsbufp->str, (int)dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); break; } @@ -2056,29 +2206,24 @@ restart: if (SEQ_TRACE_TOKEN(c_p) == NIL) *esp++ = NIL; else { + Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); + Uint sender_sz = is_immed(sender) ? 0 : size_object(sender); + ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA); + if (sender_sz) { + sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc)); + } *esp++ = make_tuple(ehp); ehp[0] = make_arityval(5); ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p); ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p); ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p); - ehp[4] = SEQ_TRACE_TOKEN_SENDER(c_p); + ehp[4] = sender; ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p); ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); ASSERT(is_immed(ehp[1])); ASSERT(is_immed(ehp[2])); ASSERT(is_immed(ehp[3])); ASSERT(is_immed(ehp[5])); - if(!is_immed(ehp[4])) { - Eterm *sender = &ehp[4]; - ehp += 6; - *sender = copy_struct(*sender, - size_object(*sender), - &ehp, - &MSO(psp)); - } - else - ehp += 6; - } break; case matchEnableTrace: @@ -2127,12 +2272,12 @@ restart: if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { + ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); ehp[1] = cp[0]; ehp[2] = cp[1]; ehp[3] = make_small((Uint) cp[2]); - ehp += 4; } break; case matchSilent: @@ -2209,8 +2354,12 @@ restart: } } break; - case matchCatch: + case matchCatch: /* Match success, now build result */ do_catch = 1; + if (in_flags & ERTS_PAM_COPY_RESULT) { + build_proc = c_p; + esdp->current_process = c_p; + } break; case matchHalt: goto success; @@ -2219,9 +2368,16 @@ restart: } } fail: +#if HALFWORD_HEAP + if (c_p_checkpoint.p) { + /* Dispose garbage built by guards on caller heap */ + heap_checkpoint_revert(&c_p_checkpoint); + c_p_checkpoint.p = NULL; + } +#endif *return_flags = 0U; - if (fail_label >= 0) { /* We failed during a "TryMeElse", - lets restart, with the next match + if (fail_label >= 0) { /* We failed during a "TryMeElse", + lets restart, with the next match program */ pc = (prog->text) + fail_label; cleanup_match_pseudo_process(mpsp, 1); @@ -2235,11 +2391,6 @@ success: erl_exit(1, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } - if (*eheap_fence != FENCE_PATTERN) { - erl_exit(1, "Eheap fence overwritten in db_prog_match after op " - "0x%08x, overwritten with 0x%08x.", save_op, - *eheap_fence); - } if (*stack_fence != FENCE_PATTERN) { erl_exit(1, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, @@ -2250,6 +2401,7 @@ success: esdp->current_process = current_scheduled; END_ATOMIC_TRACE(c_p); + return ret; #undef FAIL #undef FAIL_TERM @@ -2261,7 +2413,8 @@ success: /* * Convert a match program to a "magic" binary to return up to erlang */ -Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) { +Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) +{ return erts_mk_magic_binary_term(hpp, &MSO(p), mp); } @@ -2327,13 +2480,13 @@ void db_free_dmc_err_info(DMCErrInfo *ei){ ** Store bignum in *hpp and increase *hpp accordingly. ** *hpp is assumed to be large enough to hold the result. */ -Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) +Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr) { DeclareTmpHeapNoproc(big_tmp,2); Eterm res; Sint ires; - Eterm arg1; - Eterm arg2; + Wterm arg1; + Wterm arg2; if (is_both_small(counter,incr)) { ires = signed_val(counter) + signed_val(incr); @@ -2374,6 +2527,34 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) } } +/* Must be called to read elements after db_lookup_dbterm. +** Will decompress if needed. +** HEALFWORD_HEAP: +** Will convert from relative to Wterm format if needed. +** (but only on top level, tuples and lists will still contain rterms) +*/ +Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) +{ + Eterm elem = handle->dbterm->tpl[position]; + if (!is_header(elem)) { +#if HALFWORD_HEAP + if (!is_immed(elem) + && !handle->tb->common.compress + && !(handle->abs_vec && handle->abs_vec[position])) { + return rterm2wterm(elem, handle->dbterm->tpl); + } +#endif + return elem; + } + + ASSERT(((DbTableCommon*)handle->tb)->compress); + ASSERT(!handle->mustResize); + handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, + handle->dbterm); + handle->mustResize = 1; + return handle->dbterm->tpl[position]; +} + /* ** Update one element: ** handle: Initialized by db_lookup_dbterm() @@ -2390,14 +2571,17 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; +#if HALFWORD_HEAP + Eterm* old_base; +#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; -#ifdef DEBUG_CLONE + #ifdef DEBUG_CLONE if (handle->dbterm->debug_clone) { handle->dbterm->debug_clone[position] = newval; } -#endif + #endif return; } if (!handle->mustResize) { @@ -2406,49 +2590,78 @@ void db_do_update_element(DbUpdateHandle* handle, handle->dbterm); handle->mustResize = 1; oldval = handle->dbterm->tpl[position]; + #if HALFWORD_HEAP + old_base = NULL; + #endif } - else if (is_boxed(newval)) { - newp = boxed_val(newval); - switch (*newp & _TAG_HEADER_MASK) { - case _TAG_HEADER_POS_BIG: - case _TAG_HEADER_NEG_BIG: - case _TAG_HEADER_FLOAT: - case _TAG_HEADER_HEAP_BIN: - newval_sz = header_arity(*newp) + 1; - if (is_boxed(oldval)) { - oldp = boxed_val(oldval); - switch (*oldp & _TAG_HEADER_MASK) { - case _TAG_HEADER_POS_BIG: - case _TAG_HEADER_NEG_BIG: - case _TAG_HEADER_FLOAT: - case _TAG_HEADER_HEAP_BIN: - oldval_sz = header_arity(*oldp) + 1; - if (oldval_sz == newval_sz) { - /* "self contained" terms of same size, do memcpy */ - sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); - return; + else { + #if HALFWORD_HEAP + ASSERT(!handle->abs_vec); + old_base = handle->dbterm->tpl; + #endif + if (is_boxed(newval)) { + newp = boxed_val(newval); + switch (*newp & _TAG_HEADER_MASK) { + case _TAG_HEADER_POS_BIG: + case _TAG_HEADER_NEG_BIG: + case _TAG_HEADER_FLOAT: + case _TAG_HEADER_HEAP_BIN: + newval_sz = header_arity(*newp) + 1; + if (is_boxed(oldval)) { + oldp = boxed_val_rel(oldval,old_base); + switch (*oldp & _TAG_HEADER_MASK) { + case _TAG_HEADER_POS_BIG: + case _TAG_HEADER_NEG_BIG: + case _TAG_HEADER_FLOAT: + case _TAG_HEADER_HEAP_BIN: + oldval_sz = header_arity(*oldp) + 1; + if (oldval_sz == newval_sz) { + /* "self contained" terms of same size, do memcpy */ + sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); + return; + } + goto both_size_set; } - goto both_size_set; } + goto new_size_set; } - goto new_size_set; } } } +#if HALFWORD_HEAP + else { + old_base = (handle->tb->common.compress + || (handle->abs_vec && handle->abs_vec[position])) ? + NULL : handle->dbterm->tpl; + } +#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - - oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); + + oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; - /* write new value in old dbterm, finalize will make a flat copy */ + /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; handle->mustResize = 1; + +#if HALFWORD_HEAP + if (old_base && newval_sz > 0) { + ASSERT(!handle->tb->common.compress); + if (!handle->abs_vec) { + int i = header_arity(handle->dbterm->tpl[0]); + handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); + sys_memset(handle->abs_vec, 0, i+1); + /* abs_vec[0] not used */ + } + handle->abs_vec[position] = 1; + } +#endif } static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, @@ -2547,7 +2760,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, tpl[arity + 1] = alloc_size; tmp_offheap.first = NULL; - tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); + tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); dest->first_oh = tmp_offheap.first; for (i=1; i<=arity; i++) { if (i != tb->keypos) { @@ -2566,7 +2779,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); dest->debug_clone = dbg_top; tmp_offheap.first = dest->first_oh; - copy_struct(obj, dest->size, &dbg_top, &tmp_offheap); + copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); dest->first_oh = tmp_offheap.first; ASSERT(dbg_top == dest->debug_clone + dest->size); } @@ -2613,7 +2826,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) newp->size = size; top = newp->tpl; tmp_offheap.first = NULL; - copy_struct(obj, size, &top, &tmp_offheap); + copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); newp->first_oh = tmp_offheap.first; #ifdef DEBUG_CLONE newp->debug_clone = NULL; @@ -2652,7 +2865,7 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) top = copy_to_comp(tb, obj, newp, new_sz); ASSERT(top <= basep + new_sz); - // SVERK: realloc? + /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ return basep; } @@ -2673,6 +2886,9 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) *(handle->bp) = newp; newDbTerm = (DbTerm*) (newp + offset); newDbTerm->size = handle->new_size; +#ifdef DEBUG_CLONE + newDbTerm->debug_clone = NULL; +#endif /* make a flat copy */ @@ -2682,17 +2898,38 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) db_free_tmp_uncompressed(handle->dbterm); } else { - Eterm* top; ErlOffHeap tmp_offheap; + Eterm* tpl = handle->dbterm->tpl; + Eterm* top = newDbTerm->tpl; + tmp_offheap.first = NULL; - top = newDbTerm->tpl; - copy_struct(make_tuple(handle->dbterm->tpl), handle->new_size, - &top, &tmp_offheap); - newDbTerm->first_oh = tmp_offheap.first; -#ifdef DEBUG_CLONE - newDbTerm->debug_clone = NULL; -#endif - ASSERT((byte*)top <= (newp + alloc_sz)); + + #if HALFWORD_HEAP + if (handle->abs_vec) { + int i, arity = header_arity(handle->dbterm->tpl[0]); + + top[0] = tpl[0]; + top += arity + 1; + for (i=1; i<=arity; i++) { + Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; + + newDbTerm->tpl[i] = copy_struct_rel(tpl[i], + size_object_rel(tpl[i],src_base), + &top, &tmp_offheap, src_base, + newDbTerm->tpl); + } + newDbTerm->first_oh = tmp_offheap.first; + ASSERT((byte*)top <= (newp + alloc_sz)); + erts_free(ERTS_ALC_T_TMP, handle->abs_vec); + } + else + #endif /* HALFWORD_HEAP */ + { + copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, + &tmp_offheap, tpl, top); + newDbTerm->first_oh = tmp_offheap.first; + ASSERT((byte*)top == (newp + alloc_sz)); + } } } @@ -2705,9 +2942,9 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, hp[0] = bp->tpl[0]; *hpp += arity + 1; - hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos], - size_object(bp->tpl[tb->keypos]), - hpp, off_heap); + hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], + size_object_rel(bp->tpl[tb->keypos], bp->tpl), + hpp, off_heap, bp->tpl, NULL); for (i=arity; i>0; i--) { if (i != tb->keypos) { if (is_immed(bp->tpl[i])) { @@ -2721,7 +2958,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, } ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE - ASSERT(eq(make_tuple(hp),make_tuple(bp->debug_clone))); + ASSERT(eq_rel(make_tuple(hp),make_tuple(bp->debug_clone),bp->debug_clone)); #endif return make_tuple(hp); } @@ -2744,14 +2981,14 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, hp += extra; HRelease(p, endp, hp); #ifdef DEBUG_CLONE - ASSERT(eq(copy,obj->debug_clone[pos])); + ASSERT(eq_rel(copy, obj->debug_clone[pos], obj->debug_clone)); #endif return copy; } else { - Uint sz = size_object(obj->tpl[pos]); + Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); *hpp = HAlloc(p, sz + extra); - return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p)); + return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); } } @@ -2765,7 +3002,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) ProcBin tmp; for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) { - if ((UWord)u.voidp % sizeof(UWord) != 0) { /* unaligned ptr */ + if ((UWord)u.voidp % sizeof(Uint) != 0) { /* unaligned ptr */ sys_memcpy(&tmp, u.voidp, sizeof(tmp)); /* Warning, must pass (void*)-variable to memcpy. Otherwise it will cause Bus error on Sparc due to false compile time assumptions @@ -2961,7 +3198,7 @@ static DMCRet dmc_one_term(DMCContext *context, ** Ouch, big integer in match variable. */ Eterm *save_hp; - ASSERT(heap->data == heap->def); + ASSERT(heap->vars == heap->vars_def); sz = sz2 = sz3 = 0; for (j = 0; j < context->num_match; ++j) { sz += size_object(context->matchexpr[j]); @@ -2999,24 +3236,23 @@ static DMCRet dmc_one_term(DMCContext *context, may be atoms that changed */ context->matchexpr[j] = context->copy->mem[j]; } - heap->data = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP, - heap->size*sizeof(unsigned)); - sys_memset(heap->data, 0, - heap->size * sizeof(unsigned)); + heap->vars = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP, + heap->size*sizeof(DMCVariable)); + sys_memset(heap->vars, 0, heap->size * sizeof(DMCVariable)); DMC_CLEAR(*stack); /*DMC_PUSH(*stack,NIL);*/ DMC_CLEAR(*text); return retRestart; } - if (heap->data[n]) { /* already bound ? */ + if (heap->vars[n].is_bound) { DMC_PUSH(*text,matchCmp); DMC_PUSH(*text,n); } else { /* Not bound, bind! */ - if (n >= heap->used) - heap->used = n + 1; + if (n >= heap->vars_used) + heap->vars_used = n + 1; DMC_PUSH(*text,matchBind); DMC_PUSH(*text,n); - heap->data[n] = 1; + heap->vars[n].is_bound = 1; } } else if (c == am_Underscore) { DMC_PUSH(*text, matchSkip); @@ -3041,6 +3277,8 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*stack, c); break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): + { + Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); #if HALFWORD_HEAP { @@ -3048,25 +3286,27 @@ static DMCRet dmc_one_term(DMCContext *context, UWord u; Uint t[2]; } fiddle; - ASSERT(thing_arityval(*internal_ref_val(c)) == 3); - fiddle.t[0] = *internal_ref_val(c); - fiddle.t[1] = (Uint) internal_ref_val(c)[1]; + ASSERT(thing_arityval(ref_val[0]) == 3); + fiddle.t[0] = ref_val[0]; + fiddle.t[1] = ref_val[1]; DMC_PUSH(*text, fiddle.u); - fiddle.t[0] = (Uint) internal_ref_val(c)[2]; - fiddle.t[1] = (Uint) internal_ref_val(c)[3]; + fiddle.t[0] = ref_val[2]; + fiddle.t[1] = ref_val[3]; DMC_PUSH(*text, fiddle.u); } #else - n = thing_arityval(*internal_ref_val(c)); - DMC_PUSH(*text, *internal_ref_val(c)); - for (i = 1; i <= n; ++i) { - DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]); + n = thing_arityval(ref_val[0]); + for (i = 0; i <= n; ++i) { + DMC_PUSH(*text, ref_val[i]); } #endif break; + } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - n = thing_arityval(*big_val(c)); + { + Eterm* bval = big_val(c); + n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); #if HALFWORD_HEAP { @@ -3075,13 +3315,13 @@ static DMCRet dmc_one_term(DMCContext *context, Uint t[2]; } fiddle; ASSERT(n >= 1); - fiddle.t[0] = *big_val(c); - fiddle.t[1] = big_val(c)[1]; + fiddle.t[0] = bval[0]; + fiddle.t[1] = bval[1]; DMC_PUSH(*text, fiddle.u); for (i = 2; i <= n; ++i) { - fiddle.t[0] = big_val(c)[i]; + fiddle.t[0] = bval[i]; if (++i <= n) { - fiddle.t[1] = big_val(c)[i]; + fiddle.t[1] = bval[i]; } else { fiddle.t[1] = (Uint) 0; } @@ -3089,12 +3329,12 @@ static DMCRet dmc_one_term(DMCContext *context, } } #else - DMC_PUSH(*text, *big_val(c)); - for (i = 1; i <= n; ++i) { - DMC_PUSH(*text, (Uint) big_val(c)[i]); + for (i = 0; i <= n; ++i) { + DMC_PUSH(*text, (Uint) bval[i]); } #endif break; + } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); #if HALFWORD_HEAP @@ -3227,7 +3467,6 @@ static DMCRet dmc_list(DMCContext *context, DMC_PUSH(*text, matchConsB); } --context->stack_used; /* Two objects on stack becomes one */ - context->eheap_need += 2; return retOk; } @@ -3286,7 +3525,6 @@ static DMCRet dmc_tuple(DMCContext *context, DMC_PUSH(*text, matchMkTuple); DMC_PUSH(*text, nelems); context->stack_used -= (nelems - 1); - context->eheap_need += (nelems + 1); *constant = 0; return retOk; } @@ -3304,9 +3542,6 @@ static DMCRet dmc_whole_expression(DMCContext *context, } else { ASSERT(is_tuple(context->matchexpr [context->current_match])); - context->eheap_need += - arityval(*(tuple_val(context->matchexpr - [context->current_match]))) * 2; DMC_PUSH(*text, matchPushArrayAsList); } } else { @@ -3319,6 +3554,41 @@ static DMCRet dmc_whole_expression(DMCContext *context, return retOk; } +/* Figure out which PushV instruction to use. +*/ +static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, + DMC_STACK_TYPE(UWord) *text, Uint n) +{ + DMCVariable* v = &heap->vars[n]; + MatchOps instr = matchPushV; + + ASSERT(n < heap->vars_used && v->is_bound); + if (context->is_guard) { + #if HALFWORD_HEAP + if (!v->first_guard_label) { + v->first_guard_label = DMC_STACK_NUM(*text); + ASSERT(v->first_guard_label); + instr = matchPushVGuard; /* may be changed to PushVResult below */ + } + #endif + } + else { /* body */ + #if HALFWORD_HEAP + if (v->first_guard_label) { + /* Avoid double-copy, copy to result heap at first encounter in guard */ + DMC_POKE(*text, v->first_guard_label, matchPushVResult); + v->is_in_body = 1; + } + #endif + if (!v->is_in_body) { + instr = matchPushVResult; + v->is_in_body = 1; + } + } + DMC_PUSH(*text, instr); + DMC_PUSH(*text, n); +} + static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -3326,13 +3596,13 @@ static DMCRet dmc_variable(DMCContext *context, int *constant) { Uint n = db_is_variable(t); - ASSERT(n >= 0); - if (n >= heap->used) - RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); - if (heap->data[n] == 0U) + + if (n >= heap->vars_used || !heap->vars[n].is_bound) { RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); - DMC_PUSH(*text, matchPushV); - DMC_PUSH(*text, n); + } + + dmc_add_pushv_variant(context, heap, text, n); + ++context->stack_used; if (context->stack_used > context->stack_need) context->stack_need = context->stack_used; @@ -3351,10 +3621,9 @@ static DMCRet dmc_all_bindings(DMCContext *context, DMC_PUSH(*text, matchPushC); DMC_PUSH(*text, NIL); - for (i = heap->used - 1; i >= 0; --i) { - if (heap->data[i]) { - DMC_PUSH(*text, matchPushV); - DMC_PUSH(*text, i); + for (i = heap->vars_used - 1; i >= 0; --i) { + if (heap->vars[i].is_bound) { + dmc_add_pushv_variant(context, heap, text, i); DMC_PUSH(*text, matchConsB); heap_used += 2; } @@ -3362,7 +3631,6 @@ static DMCRet dmc_all_bindings(DMCContext *context, ++context->stack_used; if ((context->stack_used + 1) > context->stack_need) context->stack_need = (context->stack_used + 1); - context->eheap_need += heap_used; *constant = 0; return retOk; } @@ -3765,10 +4033,6 @@ static DMCRet dmc_get_seq_token(DMCContext *context, *constant = 0; DMC_PUSH(*text, matchGetSeqToken); - context->eheap_need += (6 /* A 5-tuple is built */ - + EXTERNAL_THING_HEAD_SIZE + 2 /* Sender can - be an external - pid */); if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; return retOk; @@ -4065,7 +4329,6 @@ static DMCRet dmc_caller(DMCContext *context, } *constant = 0; DMC_PUSH(*text, matchCaller); /* Creates binary */ - context->eheap_need += 4; /* A 3-tuple is built */ if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; return retOk; @@ -4663,7 +4926,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) } save_cp = p->cp; p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, &ret_flags); + res = erts_match_set_run(p, mps, arr, n, + ERTS_PAM_COPY_RESULT, &ret_flags); p->cp = save_cp; } else { n = 0; @@ -4676,11 +4940,10 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) if (is_non_value(res)) { res = am_false; } - sz = size_object(res); + sz = 0; if (ret_flags & MATCH_SET_EXCEPTION_TRACE) sz += 2; if (ret_flags & MATCH_SET_RETURN_TRACE) sz += 2; hp = HAlloc(p, 5 + sz); - res = copy_struct(res, sz, &hp, &MSO(p)); flg = NIL; if (ret_flags & MATCH_SET_EXCEPTION_TRACE) { flg = CONS(hp, am_exception_trace, flg); @@ -4735,28 +4998,24 @@ void db_free_tmp_uncompressed(DbTerm* obj) erts_free(ERTS_ALC_T_TMP, obj); } -Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog, +Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra) { Uint32 dummy; + Eterm* base; Eterm res; if (tb->compress) { obj = db_alloc_tmp_uncompressed(tb, obj); + base = NULL; } + else base = HALFWORD_HEAP ? obj->tpl : NULL; - res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, &dummy); + res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { - if (all) { - *hpp = HAlloc(c_p, obj->size + extra); - res = copy_shallow(obj->tpl, obj->size, hpp, &MSO(c_p)); - } - else { - Uint sz = size_object(res); - *hpp = HAlloc(c_p, sz + extra); - res = copy_struct(res, sz, hpp, &MSO(c_p)); - } + *hpp = HAlloc(c_p, extra); } if (tb->compress) { @@ -4767,10 +5026,11 @@ Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog, #ifdef DMC_DEBUG + /* ** Disassemble match program */ -static void db_match_dis(Binary *bp) +void db_match_dis(Binary *bp) { MatchProg *prog = Binary2MatchProg(bp); UWord *t = prog->text; @@ -4985,6 +5245,18 @@ static void db_match_dis(Binary *bp) ++t; erts_printf("PushV\t%bpu\n", n); break; + #if HALFWORD_HEAP + case matchPushVGuard: + n = (Uint) *++t; + ++t; + erts_printf("PushVGuard\t%bpu\n", n); + break; + #endif + case matchPushVResult: + n = (Uint) *++t; + ++t; + erts_printf("PushVResult\t%bpu\n", n); + break; case matchTrue: ++t; erts_printf("True\n"); @@ -5095,7 +5367,6 @@ static void db_match_dis(Binary *bp) erts_printf("}\n"); erts_printf("num_bindings: %d\n", prog->num_bindings); erts_printf("heap_size: %bpu\n", prog->heap_size); - erts_printf("eheap_offset: %bpu\n", prog->eheap_offset); erts_printf("stack_offset: %bpu\n", prog->stack_offset); erts_printf("text: 0x%08x\n", (unsigned long) prog->text); erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset); |