diff options
Diffstat (limited to 'erts/emulator/beam')
30 files changed, 424 insertions, 361 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 6ae9736141..1ca405961f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -472,9 +472,6 @@ check_process_code(Process* rp, Module* modp) for (oh = MSO(rp).first; oh; oh = oh->next) { if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { ErlFunThing* funp = (ErlFunThing*) oh; - BeamInstr* fun_code; - - fun_code = funp->fe->address; if (INSIDE((BeamInstr *) funp->fe->address)) { if (done_gc) { diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 2855241b91..8a48049921 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -267,7 +267,7 @@ erts_debug_disassemble_1(Process* p, Eterm addr) "unknown " HEXF "\n", instr); code_ptr++; } - bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len); + bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hsz = 4+4; (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index e96014c665..32ea8588d2 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1023,8 +1023,6 @@ static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) NOINLINE; -static int hibernate(Process* c_p, Eterm module, Eterm function, - Eterm args, Eterm* reg) NOINLINE; static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args) NOINLINE; static BeamInstr* apply_fun(Process* p, Eterm fun, @@ -3415,12 +3413,17 @@ void process_main(void) r(0) = nif_bif_result; CHECK_TERM(r(0)); SET_I(c_p->cp); + c_p->cp = 0; Goto(*I); } else if (c_p->freason == TRAP) { SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); r(0) = c_p->def_arg_reg[0]; x(1) = c_p->def_arg_reg[1]; x(2) = c_p->def_arg_reg[2]; + if (c_p->flags & F_HIBERNATE_SCHED) { + c_p->flags &= ~F_HIBERNATE_SCHED; + goto do_schedule; + } Dispatch(); } reg[0] = r(0); @@ -5222,7 +5225,8 @@ void process_main(void) OpCase(i_hibernate): { SWAPOUT; - if (hibernate(c_p, r(0), x(1), x(2), reg)) { + if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } else { I = handle_error(c_p, I, reg, hibernate_3); @@ -6199,8 +6203,8 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) return ep->address; } -static int -hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) +int +erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) { int arity; Eterm tmp; @@ -6275,15 +6279,17 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - c_p->status = P_WAITING; #ifdef ERTS_SMP ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); if (c_p->msg.len > 0) erts_add_to_runq(c_p); + else #endif + c_p->status = P_WAITING; } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 788cb4209c..57fe25453d 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -480,8 +480,6 @@ static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg* Rest); static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); -static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func, - GenOpArg arity, GenOpArg label); static int freeze_code(LoaderState* stp); @@ -1413,7 +1411,6 @@ static int load_code(LoaderState* stp) { int i; - int tmp; int ci; int last_func_start = 0; char* sign; @@ -1933,7 +1930,6 @@ load_code(LoaderState* stp) case 'P': /* Byte offset into tuple or stack */ case 'Q': /* Like 'P', but packable */ VerifyTag(stp, tag, TAG_u); - tmp = tmp_op->a[arg].val; code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm)); break; case 'l': /* Floating point register. */ @@ -3401,36 +3397,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, return op; } - -static GenOp* -gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg func, - GenOpArg arity, GenOpArg label) -{ - GenOp* fi; - GenOp* op; - - NEW_GENOP(stp, fi); - fi->op = genop_i_func_info_4; - fi->arity = 4; - fi->a[0].type = TAG_u; /* untagged Zero */ - fi->a[0].val = 0; - fi->a[1] = mod; - fi->a[2] = func; - fi->a[3] = arity; - - NEW_GENOP(stp, op); - op->op = genop_label_1; - op->arity = 1; - op->a[0] = label; - - fi->next = op; - op->next = NULL; - - return fi; -} - - - static GenOp* gen_make_fun2(LoaderState* stp, GenOpArg idx) { diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index bb237e378a..6b1ce823cb 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -368,7 +368,6 @@ static int demonitor(Process *c_p, Eterm ref) ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Process *rp; /* Local target process */ Eterm to = NIL; /* Monitor link traget */ - Eterm ref_p; /* Pid of this end */ DistEntry *dep = NULL; /* Target's distribution entry */ int deref_de = 0; int res; @@ -381,7 +380,6 @@ static int demonitor(Process *c_p, Eterm ref) res = ERTS_DEMONITOR_BADARG; goto done; /* Cannot be this monitor's ref */ } - ref_p = c_p->id; mon = erts_lookup_monitor(c_p->monitors, ref); if (!mon) { @@ -1091,10 +1089,20 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) BIF_RETTYPE hibernate_3(BIF_ALIST_3) { /* - * hibernate/3 is implemented as an instruction; therefore - * this function will never be called. + * hibernate/3 is usually translated to an instruction; therefore + * this function is only called from HiPE or when the call could not + * be translated. */ - BIF_ERROR(BIF_P, BADARG); + Eterm reg[3]; + + if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { + /* + * If hibernate succeeded, TRAP. The process will be suspended + * if status is P_WAITING or continue (if any message was in the queue). + */ + BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i); + } + return THE_NON_VALUE; } /**********************************************************************/ diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index a84ee7bb23..8faa09feb8 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -201,6 +201,12 @@ do { \ return THE_NON_VALUE; \ } while(0) +#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ + } while(0) + extern Export bif_return_trap_export; #ifdef DEBUG #define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \ diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 4be869f269..9486602633 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -56,7 +56,7 @@ erts_init_binary(void) */ Eterm -new_binary(Process *p, byte *buf, int len) +new_binary(Process *p, byte *buf, Uint len) { ProcBin* pb; Binary* bptr; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 02910fad90..05eddf4ae0 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -864,11 +864,15 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) #include <valgrind/valgrind.h> #include <valgrind/memcheck.h> +#ifndef HAVE_VALGRIND_PRINTF_XML +#define VALGRIND_PRINTF_XML VALGRIND_PRINTF +#endif + # define PURIFY_MSG(msg) \ do { \ char buf__[1]; size_t bufsz__ = sizeof(buf__); \ if (erts_sys_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \ - VALGRIND_PRINTF("<erlang_error_log>" \ + VALGRIND_PRINTF_XML("<erlang_error_log>" \ "%s, line %d: %s</erlang_error_log>\n", \ __FILE__, __LINE__, msg); \ } else { \ @@ -900,7 +904,6 @@ int erts_net_message(Port *prt, ErtsDistExternal ede; byte *t; Sint ctl_len; - int orig_ctl_len; Eterm arg; Eterm from, to; Eterm watcher, watched; @@ -981,7 +984,6 @@ int erts_net_message(Port *prt, PURIFY_MSG("data error"); goto data_error; } - orig_ctl_len = ctl_len; if (ctl_len > DIST_CTL_DEFAULT_SIZE) { ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm)); diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 17ff9c3b03..ce792d4d17 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -172,9 +172,17 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size); void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); +void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size); + #endif /* #if !ERTS_ALC_DO_INLINE */ +#ifndef ERTS_CACHE_LINE_SIZE +/* Assume a cache line size of 64 bytes */ +# define ERTS_CACHE_LINE_SIZE ((UWord) 64) +# define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) +#endif + #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) ERTS_ALC_INLINE @@ -234,6 +242,18 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) size); } +ERTS_ALC_INLINE +void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size) +{ + UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)); + + if (v & ERTS_CACHE_LINE_MASK) { + v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + } + ASSERT((v & ERTS_CACHE_LINE_MASK) == 0); + return (void*)v; +} + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ typedef void (*erts_alloc_verify_func_t)(Allctr_t *); @@ -241,11 +261,6 @@ typedef void (*erts_alloc_verify_func_t)(Allctr_t *); erts_alloc_verify_func_t erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr); -#ifndef ERTS_CACHE_LINE_SIZE -/* Assume a cache line size of 64 bytes */ -# define ERTS_CACHE_LINE_SIZE ((UWord) 64) -# define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) -#endif #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \ (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 4a717d7271..71206c48b2 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1545,7 +1545,7 @@ process_info_aux(Process *BIF_P, case am_backtrace: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp); - res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len); + res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hp = HAlloc(BIF_P, 3); break; @@ -1723,8 +1723,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } else if (is_list(*tp)) { #if defined(PURIFY) #define ERTS_ERROR_CHECKER_PRINTF purify_printf +#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf #elif defined(VALGRIND) #define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF +# ifndef HAVE_VALGRIND_PRINTF_XML +# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF +# else +# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML +# endif #endif int buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); @@ -1741,8 +1747,8 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } buf[buf_size - 1 - r] = '\0'; if (check_if_xml()) { - ERTS_ERROR_CHECKER_PRINTF("<erlang_info_log>" - "%s</erlang_info_log>\n", buf); + ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>" + "%s</erlang_info_log>\n", buf); } else { ERTS_ERROR_CHECKER_PRINTF("%s\n", buf); } @@ -2074,7 +2080,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); ASSERT(dsbufp && dsbufp->str); - res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len); + res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_info_dsbuf(dsbufp); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) { diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 6f8a7436d5..0174e5fc43 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -177,7 +177,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff byte* LSB; byte* MSB; Uint* hp; - Uint* hp_end; Uint words_needed; Uint actual; Uint v32; @@ -405,7 +404,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff default: words_needed = 1+WSIZE(bytes); hp = HeapOnlyAlloc(p, words_needed); - hp_end = hp + words_needed; res = bytes_to_big(LSB, bytes, sgn, hp); if (is_small(res)) { p->htop = hp; @@ -425,7 +423,6 @@ Eterm erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb) { ErlSubBin* sb; - size_t num_bytes; /* Number of bytes in binary. */ if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; @@ -435,7 +432,6 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe * From now on, we can't fail. */ - num_bytes = NBYTES(num_bits); sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE); sb->thing_word = HEADER_SUB_BIN; @@ -1557,7 +1553,6 @@ Uint32 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) { Uint bytes; - Uint bits; Uint offs; byte bigbuf[4]; byte* LSB; @@ -1567,7 +1562,6 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) ASSERT(mb->size - mb->offset >= 32); bytes = 4; - bits = 8; offs = 0; LSB = bigbuf; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 5b74240cc3..61e8a595be 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2773,17 +2773,10 @@ void init_db(void) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES, - (sizeof(erts_meta_main_tab_lock_t) - * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1))); - - if ((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0) - meta_main_tab_locks = ((erts_meta_main_tab_lock_t *) - ((((UWord) meta_main_tab_locks) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0); + meta_main_tab_locks = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, + sizeof(erts_meta_main_tab_lock_t) + * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE); for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 6cdbec3213..a59c0c258d 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -575,7 +575,7 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } -static ERTS_INLINE int cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, +static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, TreeDbTerm* obj) { return cmp_rel(key, key_base, diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index d3e31da413..0b63ab9ba0 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2162,7 +2162,8 @@ restart: case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); - *esp++ = new_binary(build_proc, (byte *)dsbufp->str, (int)dsbufp->str_len); + *esp++ = new_binary(build_proc, (byte *)dsbufp->str, + dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); break; } diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 13a73e01bb..3c0eade0d8 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -282,7 +282,7 @@ typedef struct erl_drv_entry { the port */ void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); /* called when we have input from one of - the driver's handles) */ + the driver's handles */ void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event); /* called when output is possible to one of the driver's handles */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 2aa932e7d1..c30d67ac88 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,6 +33,7 @@ #include "erl_gc.h" #if HIPE #include "hipe_stack.h" +#include "hipe_mode_switch.h" #endif #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 @@ -454,7 +455,6 @@ erts_garbage_collect_hibernate(Process* p) Eterm* heap; Eterm* htop; Rootset rootset; - int n; char* src; Uint src_size; Uint actual_size; @@ -485,7 +485,10 @@ erts_garbage_collect_hibernate(Process* p) sizeof(Eterm)*heap_size); htop = heap; - n = setup_rootset(p, p->arg_reg, p->arity, &rootset); + (void) setup_rootset(p, p->arg_reg, p->arity, &rootset); +#if HIPE + hipe_empty_nstack(p); +#endif src = (char *) p->heap; src_size = (char *) p->htop - src; diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 0372e6850d..cdb06d4458 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/erl_nmgc.c b/erts/emulator/beam/erl_nmgc.c index 626d4e295a..d7bfb2ab12 100644 --- a/erts/emulator/beam/erl_nmgc.c +++ b/erts/emulator/beam/erl_nmgc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,7 +26,6 @@ #include "erl_nmgc.h" #include "erl_debug.h" #if HIPE -#include "hipe_bif0.h" /* for hipe_constants_{start,next} */ #include "hipe_stack.h" #endif diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 714b4ea7dd..3e2c5f07ab 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -102,6 +102,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void) { + ERTS_THR_MEMORY_BARRIER; return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0; } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index d9f132f067..b71404fd27 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -216,233 +216,289 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) } +#define PRT_BAR ((Eterm) 0) +#define PRT_COMMA ((Eterm) 1) +#define PRT_CLOSE_LIST ((Eterm) 2) +#define PRT_CLOSE_TUPLE ((Eterm) 3) +#define PRT_TERM ((Eterm) 4) +#define PRT_ONE_CONS ((Eterm) 5) +#define PRT_PATCH_FUN_SIZE ((Eterm) 6) +#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */ static int print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { + DECLARE_WSTACK(s); int res; int i; + Eterm val; Uint32 *ref_num; Eterm* nobj; +#if HALFWORD_HEAP + UWord wobj; +#endif res = 0; - if ((*dcount)-- <= 0) - return res; - -#ifdef HYBRID___NOT_ACTIVE - /* Color coded output based on memory location */ - if(ptr_val(obj) >= global_heap && ptr_val(obj) < global_hend) - PRINT_STRING(res, fn, arg, "\033[32m"); -#ifdef INCREMENTAL - else if(ptr_val(obj) >= inc_fromspc && ptr_val(obj) < inc_fromend) - PRINT_STRING(res, fn, arg, "\033[33m"); + goto L_jump_start; + + L_outer_loop: + while (!WSTACK_ISEMPTY(s)) { + switch (val = WSTACK_POP(s)) { + case PRT_COMMA: + PRINT_CHAR(res, fn, arg, ','); + goto L_outer_loop; + case PRT_BAR: + PRINT_CHAR(res, fn, arg, '|'); + goto L_outer_loop; + case PRT_CLOSE_LIST: + PRINT_CHAR(res, fn, arg, ']'); + goto L_outer_loop; + case PRT_CLOSE_TUPLE: + PRINT_CHAR(res, fn, arg, '}'); + goto L_outer_loop; + default: +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); #endif - else if(IS_CONST(obj)) - PRINT_STRING(res, fn, arg, "\033[34m"); - else - PRINT_STRING(res, fn, arg, "\033[31m"); + switch (val) { + case PRT_TERM: + break; + case PRT_ONE_CONS: + L_print_one_cons: + { + Eterm* cons = list_val(obj); + Eterm tl; + + obj = CAR(cons); + tl = CDR(cons); + if (is_not_nil(tl)) { + if (is_list(tl)) { + WSTACK_PUSH(s, tl); + WSTACK_PUSH(s, PRT_ONE_CONS); + WSTACK_PUSH(s, PRT_COMMA); + } else { + WSTACK_PUSH(s, tl); + WSTACK_PUSH(s, PRT_TERM); + WSTACK_PUSH(s, PRT_BAR); + } + } + } + break; + case PRT_LAST_ARRAY_ELEMENT: + { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else + Eterm* ptr = (Eterm *) obj; #endif + obj = *ptr; + } + break; + default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ + { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else + Eterm* ptr = (Eterm *) obj; +#endif + obj = *ptr++; + WSTACK_PUSH(s, (UWord) ptr); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, PRT_COMMA); + } + break; + } + break; + } - if (is_CP(obj)) { - PRINT_STRING(res, fn, arg, "<cp/header:"); - PRINT_POINTER(res, fn, arg, cp_val(obj)); - PRINT_CHAR(res, fn, arg, '>'); - return res; - } + L_jump_start: - switch (tag_val_def(obj)) { - case NIL_DEF: - PRINT_STRING(res, fn, arg, "[]"); - break; - case ATOM_DEF: { - int tres = print_atom_name(fn, arg, obj, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; - break; - } - case SMALL_DEF: - PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); - break; - case BIG_DEF: { - int print_res; - char def_buf[64]; - char *buf, *big_str; - Uint sz = (Uint) big_decimal_estimate(obj); - sz++; - if (sz <= 64) - buf = &def_buf[0]; - else - buf = erts_alloc(ERTS_ALC_T_TMP, sz); - big_str = erts_big_to_string(obj, buf, sz); - print_res = erts_printf_string(fn, arg, big_str); - if (buf != &def_buf[0]) - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (print_res < 0) - return print_res; - res += print_res; - break; - } - case REF_DEF: - case EXTERNAL_REF_DEF: - PRINT_STRING(res, fn, arg, "#Ref<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) ref_channel_no(obj)); - ref_num = ref_numbers(obj); - for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + if ((*dcount)-- <= 0) + goto L_done; + + if (is_CP(obj)) { + PRINT_STRING(res, fn, arg, "<cp/header:"); + PRINT_POINTER(res, fn, arg, cp_val(obj)); + PRINT_CHAR(res, fn, arg, '>'); + goto L_done; } - PRINT_CHAR(res, fn, arg, '>'); - break; - case PID_DEF: - case EXTERNAL_PID_DEF: - PRINT_CHAR(res, fn, arg, '<'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_channel_no(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_number(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_serial(obj)); - PRINT_CHAR(res, fn, arg, '>'); - break; - case PORT_DEF: - case EXTERNAL_PORT_DEF: - PRINT_STRING(res, fn, arg, "#Port<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_channel_no(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_number(obj)); - PRINT_CHAR(res, fn, arg, '>'); - break; - case LIST_DEF: - if (is_printable_string(obj)) { - int c; - PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val(obj); - while (1) { - if ((*dcount)-- <= 0) - return res; - c = signed_val(*nobj++); - if (c == '\n') - PRINT_STRING(res, fn, arg, "\\n"); - else { - if (c == '"') - PRINT_CHAR(res, fn, arg, '\\'); - PRINT_CHAR(res, fn, arg, (char) c); - } - if (is_not_list(*nobj)) - break; - nobj = list_val(*nobj); - } - PRINT_CHAR(res, fn, arg, '"'); - } else { - PRINT_CHAR(res, fn, arg, '['); - nobj = list_val(obj); - while (1) { - int tres = print_term(fn, arg, *nobj++, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; - if (is_not_list(*nobj)) - break; - PRINT_CHAR(res, fn, arg, ','); - nobj = list_val(*nobj); - } - if (is_not_nil(*nobj)) { - int tres; - PRINT_CHAR(res, fn, arg, '|'); - tres = print_term(fn, arg, *nobj, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; + + switch (tag_val_def(obj)) { + case NIL_DEF: + PRINT_STRING(res, fn, arg, "[]"); + break; + case ATOM_DEF: { + int tres = print_atom_name(fn, arg, obj, dcount); + if (tres < 0) { + res = tres; + goto L_done; } - PRINT_CHAR(res, fn, arg, ']'); - } - break; - case TUPLE_DEF: - nobj = tuple_val(obj); /* pointer to arity */ - i = arityval(*nobj); /* arity */ - PRINT_CHAR(res, fn, arg, '{'); - while (i--) { - int tres = print_term(fn, arg, *++nobj, dcount); - if (tres < 0) - return tres; res += tres; if (*dcount <= 0) - return res; - if (i >= 1) - PRINT_CHAR(res, fn, arg, ','); - } - PRINT_CHAR(res, fn, arg, '}'); - break; - case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(obj, ff); - PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); + goto L_done; + break; } - break; - case BINARY_DEF: - { - ProcBin* pb = (ProcBin *) binary_val(obj); - if (pb->size == 1) - PRINT_STRING(res, fn, arg, "<<1 byte>>"); - else { - PRINT_STRING(res, fn, arg, "<<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); - PRINT_STRING(res, fn, arg, " bytes>>"); + case SMALL_DEF: + PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); + break; + case BIG_DEF: { + int print_res; + char def_buf[64]; + char *buf, *big_str; + Uint sz = (Uint) big_decimal_estimate(obj); + sz++; + if (sz <= 64) + buf = &def_buf[0]; + else + buf = erts_alloc(ERTS_ALC_T_TMP, sz); + big_str = erts_big_to_string(obj, buf, sz); + print_res = erts_printf_string(fn, arg, big_str); + if (buf != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, (void *) buf); + if (print_res < 0) { + res = print_res; + goto L_done; } + res += print_res; + break; } - break; - case EXPORT_DEF: - { - Export* ep = *((Export **) (export_val(obj) + 1)); - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); - - PRINT_STRING(res, fn, arg, "#Fun<"); - PRINT_BUF(res, fn, arg, module->name, module->len); + case REF_DEF: + case EXTERNAL_REF_DEF: + PRINT_STRING(res, fn, arg, "#Ref<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) ref_channel_no(obj)); + ref_num = ref_numbers(obj); + for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { + PRINT_CHAR(res, fn, arg, '.'); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + } + PRINT_CHAR(res, fn, arg, '>'); + break; + case PID_DEF: + case EXTERNAL_PID_DEF: + PRINT_CHAR(res, fn, arg, '<'); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_channel_no(obj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_BUF(res, fn, arg, name->name, name->len); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_number(obj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) ep->code[2]); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_serial(obj)); PRINT_CHAR(res, fn, arg, '>'); - } - break; - case FUN_DEF: - { - ErlFunThing *funp = (ErlFunThing *) fun_val(obj); - Atom *ap = atom_tab(atom_val(funp->fe->module)); - - PRINT_STRING(res, fn, arg, "#Fun<"); - PRINT_BUF(res, fn, arg, ap->name, ap->len); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_index); + break; + case PORT_DEF: + case EXTERNAL_PORT_DEF: + PRINT_STRING(res, fn, arg, "#Port<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) port_channel_no(obj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_uniq); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) port_number(obj)); PRINT_CHAR(res, fn, arg, '>'); + break; + case LIST_DEF: + if (is_printable_string(obj)) { + int c; + PRINT_CHAR(res, fn, arg, '"'); + nobj = list_val(obj); + while (1) { + if ((*dcount)-- <= 0) + goto L_done; + c = signed_val(*nobj++); + if (c == '\n') + PRINT_STRING(res, fn, arg, "\\n"); + else { + if (c == '"') + PRINT_CHAR(res, fn, arg, '\\'); + PRINT_CHAR(res, fn, arg, (char) c); + } + if (is_not_list(*nobj)) + break; + nobj = list_val(*nobj); + } + PRINT_CHAR(res, fn, arg, '"'); + } else { + PRINT_CHAR(res, fn, arg, '['); + WSTACK_PUSH(s,PRT_CLOSE_LIST); + goto L_print_one_cons; + } + break; + case TUPLE_DEF: + nobj = tuple_val(obj); /* pointer to arity */ + i = arityval(*nobj); /* arity */ + PRINT_CHAR(res, fn, arg, '{'); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + ++nobj; + if (i > 0) { + WSTACK_PUSH(s, (UWord) nobj); + WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1); + } + break; + case FLOAT_DEF: { + FloatDef ff; + GET_DOUBLE(obj, ff); + PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); + } + break; + case BINARY_DEF: + { + ProcBin* pb = (ProcBin *) binary_val(obj); + if (pb->size == 1) + PRINT_STRING(res, fn, arg, "<<1 byte>>"); + else { + PRINT_STRING(res, fn, arg, "<<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); + PRINT_STRING(res, fn, arg, " bytes>>"); + } + } + break; + case EXPORT_DEF: + { + Export* ep = *((Export **) (export_val(obj) + 1)); + Atom* module = atom_tab(atom_val(ep->code[0])); + Atom* name = atom_tab(atom_val(ep->code[1])); + + PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_BUF(res, fn, arg, module->name, module->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_BUF(res, fn, arg, name->name, name->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) ep->code[2]); + PRINT_CHAR(res, fn, arg, '>'); + } + break; + case FUN_DEF: + { + ErlFunThing *funp = (ErlFunThing *) fun_val(obj); + Atom *ap = atom_tab(atom_val(funp->fe->module)); + + PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_BUF(res, fn, arg, ap->name, ap->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) funp->fe->old_index); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) funp->fe->old_uniq); + PRINT_CHAR(res, fn, arg, '>'); + } + break; + default: + PRINT_STRING(res, fn, arg, "<unknown:"); + PRINT_POINTER(res, fn, arg, (UWord) obj); + PRINT_CHAR(res, fn, arg, '>'); + break; } - break; - default: - PRINT_STRING(res, fn, arg, "<unknown:"); - PRINT_POINTER(res, fn, arg, (UWord) obj); - PRINT_CHAR(res, fn, arg, '>'); - break; } + L_done: + + DESTROY_WSTACK(s); return res; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4d6e982325..2efff4bf10 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -622,14 +622,10 @@ init_misc_aux_work(void) init_misc_aux_work_alloc(); - misc_aux_work_queues = erts_alloc(ERTS_ALC_T_MISC_AUX_WORK_Q, - (sizeof(erts_algnd_misc_aux_work_q_t) - *(erts_no_schedulers+1))); - if ((((UWord) misc_aux_work_queues) & ERTS_CACHE_LINE_MASK) != 0) - misc_aux_work_queues = ((erts_algnd_misc_aux_work_q_t *) - ((((UWord) misc_aux_work_queues) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); + misc_aux_work_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, + erts_no_schedulers * + sizeof(erts_algnd_misc_aux_work_q_t)); for (ix = 0; ix < erts_no_schedulers; ix++) { erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, @@ -815,15 +811,31 @@ erts_active_schedulers(void) return as; } +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_sys_scheduling(void) +{ + erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); +} + +static ERTS_INLINE int +try_set_sys_scheduling(void) +{ + return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); +} + +#endif + static ERTS_INLINE int prepare_for_sys_schedule(void) { #ifdef ERTS_SMP while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic32_xchg(&doing_sys_schedule, 1)) { + && try_set_sys_scheduling()) { if (!erts_port_task_have_outstanding_io_tasks()) return 1; - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); } return 0; #else @@ -1153,7 +1165,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * call erl_sys_schedule() until it is handled. */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... @@ -1172,7 +1184,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * sleep in erl_sys_schedule(). */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise @@ -1226,7 +1238,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_woken: erts_smp_runq_lock(rq); sys_locked_woken: - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); sched_active_sys(esdp->no, rq); @@ -1255,7 +1267,6 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) static void wake_scheduler(ErtsRunQueue *rq, int incq, int one) { - int res; ErtsSchedulerSleepInfo *ssi; ErtsSchedulerSleepList *sl; @@ -1286,9 +1297,9 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) if (ssi->next) ssi->next->prev = ssi->prev; - res = sl->list != NULL; erts_smp_spin_unlock(&sl->lock); + ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); @@ -1298,6 +1309,8 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) else { sl->list = NULL; erts_smp_spin_unlock(&sl->lock); + + ERTS_THR_MEMORY_BARRIER; do { ErtsSchedulerSleepInfo *wake_ssi = ssi; ssi = ssi->next; @@ -2515,16 +2528,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) (mrq ? no_schedulers : 1); - erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, - (sizeof(ErtsAlignedRunQueue)*(n+1))); - if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) != 0) - erts_aligned_run_queues = ((ErtsAlignedRunQueue *) - ((((UWord) erts_aligned_run_queues) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0); - + erts_aligned_run_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, + sizeof(ErtsAlignedRunQueue) * n); #ifdef ERTS_SMP erts_smp_atomic32_init(&no_empty_run_queues, 0); #endif @@ -2619,14 +2625,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP /* Create and initialize scheduler sleep info */ - aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, - (sizeof(ErtsAlignedSchedulerSleepInfo) - *(n+1))); - if ((((UWord) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) - aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) - ((((UWord) aligned_sched_sleep_info) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); + aligned_sched_sleep_info = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, + n * sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < n; ix++) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); #if 0 /* no need to initialize these... */ @@ -2641,16 +2643,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) /* Create and initialize scheduler specific data */ - erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, - (sizeof(ErtsAlignedSchedulerData) - *(n+1))); - if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) != 0) - erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) - ((((UWord) erts_aligned_scheduler_data) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0); + erts_aligned_scheduler_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + n*sizeof(ErtsAlignedSchedulerData)); for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); @@ -2860,7 +2855,6 @@ resume_process(Process *p) return; switch(p->rstatus) { case P_RUNABLE: - *statusp = P_WAITING; /* make erts_add_to_runq work */ erts_add_to_runq(p); break; case P_WAITING: @@ -4656,7 +4650,7 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - ASSERT(p->status != P_SUSPENDED); + ASSERT(p->rcount == 0); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; return NULL; @@ -4667,9 +4661,8 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); #ifndef ERTS_SMP /* Never schedule a suspended process (ok in smp case) */ - ASSERT(p->status != P_SUSPENDED); + ASSERT(p->rcount == 0); add_runq = runq; - #else ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); if (p->bound_runq) { @@ -5169,7 +5162,7 @@ Process *schedule(Process *p, int calls) handle_pending_suspend(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - || p->status != P_SUSPENDED); + || p->rcount == 0); } #endif erts_smp_runq_lock(rq); @@ -5371,7 +5364,7 @@ Process *schedule(Process *p, int calls) if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP erts_smp_runq_lock(rq); - erts_smp_atomic32_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); goto continue_check_activities_to_run; #else if (!runnable) @@ -7614,10 +7607,28 @@ timeout_proc(Process* p) p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; - if (p->status == P_WAITING) - erts_add_to_runq(p); - if (p->status == P_SUSPENDED) + switch (p->status) { + case P_GARBING: + switch (p->gcstatus) { + case P_SUSPENDED: + goto suspended; + case P_WAITING: + goto waiting; + default: + break; + } + break; + case P_WAITING: + waiting: + erts_add_to_runq(p); + break; + case P_SUSPENDED: + suspended: p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */ + break; + default: + break; + } } @@ -7703,7 +7714,7 @@ erts_program_counter_info(int to, void *to_arg, Process *p) * only cause problems. */ for (i = 0; i < p->arity; i++) - erts_print(to, to_arg, " %T\n", p->arg_reg[i]); + erts_print(to, to_arg, " %.*T\n", INT_MAX, p->arg_reg[i]); } } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index e871a9834a..334ae5573f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -895,6 +895,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ +#define F_HIBERNATE_SCHED (1 << 11) /* Schedule out after hibernate op */ /* process trace_flags */ #define F_SENSITIVE (1 << 0) @@ -1592,7 +1593,9 @@ ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { - erts_aint32_t flags = erts_smp_atomic32_read(&ssi->flags); + erts_aint32_t flags; + ERTS_THR_MEMORY_BARRIER; + flags = erts_smp_atomic32_read(&ssi->flags); ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) || (flags & ERTS_SSI_FLG_WAITING)); if (flags & ERTS_SSI_FLG_SLEEPING) { diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 93d8ea4cb4..d0ad73cd81 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 545b345a71..dacf228e92 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -902,7 +902,6 @@ static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos, static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3) { Eterm *real_bin; - Sint need; byte* bytes; Eterm rest_term; int left, sleft; @@ -918,7 +917,6 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3) ASSERT(is_binary(BIF_ARG_1)); real_bin = binary_val(BIF_ARG_1); ASSERT(*real_bin == HEADER_PROC_BIN); - need = ((ProcBin *) real_bin)->val->orig_size; pos = (int) binary_size(BIF_ARG_1); bytes = binary_bytes(BIF_ARG_1); sleft = left = allowed_iterations(BIF_P); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 328aa2be6a..1a102f7187 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1264,7 +1264,7 @@ external_size_1(Process* p, Eterm Term) Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { - int size; + Uint size; Eterm bin; size_t real_size; byte* endp; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bd540eaaa6..432bdd705b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -834,7 +834,7 @@ do { \ void erts_emasculate_writable_binary(ProcBin* pb); Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, int); -Eterm new_binary(Process*, byte*, int); +Eterm new_binary(Process*, byte*, Uint); Eterm erts_realloc_binary(Eterm bin, size_t size); /* erl_bif_info.c */ @@ -1700,6 +1700,7 @@ Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(int to, void *arg, Process *c_p); +int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); #define seq_trace_output(token, msg, type, receiver, process) \ seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) #define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index f21a96c754..2e884a350e 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1226,7 +1226,6 @@ void init_io(void) { int i; ErlDrvEntry** dp; - ErlDrvEntry* drv; char maxports[21]; /* enough for any 64-bit integer */ size_t maxportssize = sizeof(maxports); Uint ports_bits = ERTS_PORTS_BITS; @@ -1309,10 +1308,8 @@ void init_io(void) init_driver(&fd_driver, &fd_driver_entry, NULL); init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); init_driver(&spawn_driver, &spawn_driver_entry, NULL); - for (dp = driver_tab; *dp != NULL; dp++) { - drv = *dp; + for (dp = driver_tab; *dp != NULL; dp++) erts_add_driver_entry(*dp, NULL, 1); - } erts_smp_tsd_set(driver_list_lock_status_key, NULL); erts_smp_mtx_unlock(&erts_driver_list_lock); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index e861f97e7a..6caa1e0b2d 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1066,7 +1066,7 @@ is_function f y is_function f r is_function Fail=f c => jump Fail -func_info M=a F=a A=u | label L => gen_func_info(M, F, A, L) +func_info M F A => i_func_info u M F A # ================================================================ # New bit syntax matching (R11B). diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index ff828ae889..e64c43de6e 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -538,8 +538,8 @@ typedef struct preload { */ typedef struct _SysDriverOpts { - int ifd; /* Input file descriptor (fd driver). */ - int ofd; /* Outputfile descriptor (fd driver). */ + Uint ifd; /* Input file descriptor (fd driver). */ + Uint ofd; /* Outputfile descriptor (fd driver). */ int packet_bytes; /* Number of bytes in packet header. */ int read_write; /* Read and write bits. */ int use_stdio; /* Use standard I/O: TRUE or FALSE. */ diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index c65cc37fc6..a00faff912 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in |