diff options
Diffstat (limited to 'erts/emulator/beam')
33 files changed, 1189 insertions, 655 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index dfc3cde6a7..e2a79d6e4f 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -322,7 +322,7 @@ init_atom_table(void) text_list = NULL; erts_index_init(ERTS_ALC_T_ATOM_TABLE, &erts_atom_table, - "atom_tab", ATOM_SIZE, ATOM_LIMIT, f); + "atom_tab", ATOM_SIZE, erts_atom_table_size, f); more_atom_space(); /* Ordinary atoms */ diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index e7e0dc440d..cb245a87b1 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -28,6 +28,17 @@ #define MAX_ATOM_LENGTH 255 #define ATOM_LIMIT (1024*1024) +#define MIN_ATOM_TABLE_SIZE 8192 + +#ifndef ARCH_32 +/* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an + unsigned 32 bit integer. See external.c(erts_encode_ext_dist_header_setup) + for more details. */ +#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (1UL << 32)) ? MAX_ATOM_INDEX + 1 : (1UL << 32)) +#else +#define MAX_ATOM_TABLE_SIZE (MAX_ATOM_INDEX + 1) +#endif + /* * Atom entry. diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 04eac2d807..57c8b08223 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 1996-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1996-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -303,6 +303,7 @@ atom messages atom meta atom meta_match_spec atom min_heap_size +atom min_bin_vheap_size atom minor_version atom Minus='-' atom module @@ -446,6 +447,7 @@ atom running atom running_ports atom running_procs atom runtime +atom safe atom save_calls atom scheduler atom scheduler_id diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index dcaa43b51c..13757b7d1c 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -286,6 +286,15 @@ extern int count_instructions; #endif +#ifdef FORCE_HEAP_FRAGS +# define HEAP_SPACE_VERIFIED(Words) do { \ + c_p->space_verified = (Words); \ + c_p->space_verified_from = HTOP; \ + }while(0) +#else +# define HEAP_SPACE_VERIFIED(Words) ((void)0) +#endif + #define PRE_BIF_SWAPOUT(P) \ HEAP_TOP((P)) = HTOP; \ (P)->stop = E; \ @@ -411,6 +420,7 @@ extern int count_instructions; r(0) = reg[0]; \ SWAPIN; \ } \ + HEAP_SPACE_VERIFIED(need); \ } while (0) @@ -432,6 +442,7 @@ extern int count_instructions; r(0) = reg[0]; \ SWAPIN; \ } \ + HEAP_SPACE_VERIFIED(need); \ } while (0) /* @@ -456,6 +467,7 @@ extern int count_instructions; Extra = reg[Live]; \ SWAPIN; \ } \ + HEAP_SPACE_VERIFIED(need); \ } while (0) #ifdef HYBRID @@ -832,6 +844,7 @@ extern int count_instructions; LIGHT_SWAPOUT; \ _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb); \ LIGHT_SWAPIN; \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ else { Store(_result, Dst); } \ } while (0) @@ -845,6 +858,7 @@ extern int count_instructions; LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb); \ LIGHT_SWAPIN; \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ else { Store(_result, Dst); } \ } while (0) @@ -859,6 +873,7 @@ extern int count_instructions; LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb); \ LIGHT_SWAPIN; \ + HEAP_SPACE_VERIFIED(0); \ if (is_non_value(_result)) { Fail; } \ else { Store(_result, Dst); } \ } while (0) @@ -873,9 +888,12 @@ extern int count_instructions; LIGHT_SWAPOUT; \ _result = erts_bs_get_binary_all_2(c_p, _mb); \ LIGHT_SWAPIN; \ + HEAP_SPACE_VERIFIED(0); \ ASSERT(is_value(_result)); \ Store(_result, Dst); \ - } else { Fail; } \ + } else { \ + HEAP_SPACE_VERIFIED(0); \ + Fail; } \ } while (0) #define BsSkipBits2(Ms, Bits, Unit, Fail) \ @@ -974,10 +992,6 @@ static int hibernate(Process* c_p, Eterm module, Eterm function, static Eterm* call_fun(Process* p, int arity, Eterm* reg, Eterm args); static Eterm* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg); static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free); -static BIF_RETTYPE nif_dispatcher_0(Process* p, Uint* I); -static BIF_RETTYPE nif_dispatcher_1(Process* p, Eterm arg1, Uint* I); -static BIF_RETTYPE nif_dispatcher_2(Process* p, Eterm arg1, Eterm arg2, Uint* I); -static BIF_RETTYPE nif_dispatcher_3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3, Uint* I); #if defined(_OSE_) || defined(VXWORKS) static int init_done; @@ -1364,6 +1378,7 @@ void process_main(void) */ c_p->cp = 0; CHECK_TERM(r(0)); + HEAP_SPACE_VERIFIED(0); Goto(*I); } @@ -2383,6 +2398,7 @@ void process_main(void) if (is_big(tmp_arg1)) { HTOP += bignum_header_arity(*HTOP) + 1; } + HEAP_SPACE_VERIFIED(0); if (is_nil(tmp_arg1)) { /* * This result must have been only slight larger @@ -2949,11 +2965,38 @@ void process_main(void) OpCase(call_nif): { - static void* const dispatchers[4] = { - nif_dispatcher_0, nif_dispatcher_1, nif_dispatcher_2, nif_dispatcher_3 - }; - BifFunction vbf = dispatchers[I[-1]]; - goto apply_bif_or_nif; + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: priv_data pointer + */ + BifFunction vbf; + + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + PROCESS_MAIN_CHK_LOCKS(c_p); + tmp_arg2 = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + { + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + erts_pre_nif(&env, c_p, (void*)I[2]); + reg[0] = r(0); + tmp_arg1 = (*fp)(&env, tmp_arg2, reg); + erts_post_nif(&env); + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); + PROCESS_MAIN_CHK_LOCKS(c_p); + goto apply_bif_or_nif_epilogue; OpCase(apply_bif): /* @@ -2966,17 +3009,15 @@ void process_main(void) * code[3]: &&apply_bif * code[4]: Function pointer to BIF function */ - vbf = (BifFunction) Arg(0); - apply_bif_or_nif: c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ c_p->i = I; /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ - SWAPOUT; c_p->fcalls = FCALLS - 1; + vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); tmp_arg2 = I[-1]; ASSERT(tmp_arg2 <= 3); @@ -3019,6 +3060,7 @@ void process_main(void) break; } } +apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); if (c_p->mbuf) { @@ -3203,6 +3245,7 @@ void process_main(void) sb->orig = new_binary; new_binary = make_binary(sb); } + HEAP_SPACE_VERIFIED(0); StoreBifResult(2, new_binary); } else { Binary* bptr; @@ -3690,6 +3733,7 @@ void process_main(void) *dst = *ms; *HTOP = HEADER_BIN_MATCHSTATE(slots); HTOP += wordsneeded; + HEAP_SPACE_VERIFIED(0); StoreResult(make_matchstate(dst), Arg(3)); } } else if (is_binary_header(header)) { @@ -3703,6 +3747,7 @@ void process_main(void) #endif result = erts_bs_start_match_2(c_p, tmp_arg1, slots); HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); if (is_non_value(result)) { ClauseFail(); } else { @@ -3895,6 +3940,7 @@ void process_main(void) TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); _result = uint_to_big((Uint) _integer, HTOP); HTOP += BIG_UINT_HEAP_SIZE; + HEAP_SPACE_VERIFIED(0); } #endif StoreBifResult(2, _result); @@ -3960,6 +4006,7 @@ void process_main(void) LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb); LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); if (is_non_value(result)) { ClauseFail(); } @@ -3987,6 +4034,7 @@ void process_main(void) LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, size, flags, mb); LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); if (is_non_value(result)) { ClauseFail(); } @@ -5245,6 +5293,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, * The Bif does not really exist (no BIF entry). It is a * TRAP and traps are called through apply_bif, which also * sets c_p->current (luckily). + * OR it is a NIF called by call_nif where current is also set. */ ASSERT(c_p->current); s->current = c_p->current; @@ -6148,51 +6197,3 @@ erts_current_reductions(Process *current, Process *p) } } -static BIF_RETTYPE nif_dispatcher_0(Process* p, Uint* I) -{ - typedef Eterm NifF(struct enif_environment_t*); - NifF* fp = (NifF*) I[1]; - struct enif_environment_t env; - Eterm ret; - erts_pre_nif(&env, p, (void*)I[2]); - ret = (*fp)(&env); - erts_post_nif(&env); - return ret; -} - -static BIF_RETTYPE nif_dispatcher_1(Process* p, Eterm arg1, Uint* I) -{ - typedef Eterm NifF(struct enif_environment_t*, Eterm); - NifF* fp = (NifF*) I[1]; - struct enif_environment_t env; - Eterm ret; - erts_pre_nif(&env, p, (void*)I[2]); - ret = (*fp)(&env, arg1); - erts_post_nif(&env); - return ret; -} - -static BIF_RETTYPE nif_dispatcher_2(Process* p, Eterm arg1, Eterm arg2, Uint* I) -{ - typedef Eterm NifF(struct enif_environment_t*, Eterm, Eterm); - NifF* fp = (NifF*) I[1]; - struct enif_environment_t env; - Eterm ret; - erts_pre_nif(&env, p, (void*)I[2]); - ret = (*fp)(&env, arg1, arg2); - erts_post_nif(&env); - return ret; -} - -static BIF_RETTYPE nif_dispatcher_3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3, Uint* I) -{ - typedef Eterm NifF(struct enif_environment_t*, Eterm, Eterm, Eterm); - NifF* fp = (NifF*) I[1]; - struct enif_environment_t env; - Eterm ret; - erts_pre_nif(&env, p, (void*)I[2]); - ret = (*fp)(&env, arg1, arg2, arg3); - erts_post_nif(&env); - return ret; -} - diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 47dd98117d..99fab28dce 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -5090,6 +5090,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_COMPILE_PTR] = 0; code[MI_COMPILE_SIZE_ON_HEAP] = 0; code[MI_NUM_BREAKPOINTS] = 0; + code[MI_ON_LOAD_FUNCTION_PTR] = 0; ci = MI_FUNCTIONS + n + 1; /* diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 74b231d56d..9c8c0df9f0 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -807,11 +807,12 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) /* * Store default values for options. */ - so.flags = SPO_USE_ARGS; - so.min_heap_size = H_MIN_SIZE; - so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); - so.scheduler = 0; + so.flags = SPO_USE_ARGS; + so.min_heap_size = H_MIN_SIZE; + so.min_vheap_size = BIN_VH_MIN_SIZE; + so.priority = PRIORITY_NORMAL; + so.max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + so.scheduler = 0; /* * Walk through the option list. @@ -850,6 +851,15 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) } else { so.min_heap_size = erts_next_heap_size(min_heap_size, 0); } + } else if (arg == am_min_bin_vheap_size && is_small(val)) { + Sint min_vheap_size = signed_val(val); + if (min_vheap_size < 0) { + goto error; + } else if (min_vheap_size < BIN_VH_MIN_SIZE) { + so.min_vheap_size = BIN_VH_MIN_SIZE; + } else { + so.min_vheap_size = erts_next_heap_size(min_vheap_size, 0); + } } else if (arg == am_fullsweep_after && is_small(val)) { Sint max_gen_gcs = signed_val(val); if (max_gen_gcs < 0) { @@ -1485,6 +1495,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } BIF_RET(old_value); } + else if (BIF_ARG_1 == am_min_bin_vheap_size) { + Sint i; + if (!is_small(BIF_ARG_2)) { + goto error; + } + i = signed_val(BIF_ARG_2); + if (i < 0) { + goto error; + } + old_value = make_small(BIF_P->min_vheap_size); + if (i < BIN_VH_MIN_SIZE) { + BIF_P->min_vheap_size = BIN_VH_MIN_SIZE; + } else { + BIF_P->min_vheap_size = erts_next_heap_size(i, 0); + } + BIF_RET(old_value); + } else if (BIF_ARG_1 == am_sensitive) { Uint is_sensitive; if (BIF_ARG_2 == am_true) { @@ -3736,10 +3763,35 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { int oval = H_MIN_SIZE; + if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) { goto error; } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_block_system(0); + H_MIN_SIZE = erts_next_heap_size(n, 0); + + erts_smp_release_system(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + BIF_RET(make_small(oval)); + } else if (BIF_ARG_1 == am_min_bin_vheap_size) { + int oval = BIN_VH_MIN_SIZE; + + if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) { + goto error; + } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_block_system(0); + + BIN_VH_MIN_SIZE = erts_next_heap_size(n, 0); + + erts_smp_release_system(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_display_items) { int oval = display_items; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 85a729208f..b6fa06354a 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 1996-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1996-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -755,6 +755,11 @@ bif erlang:call_on_load_function/1 bif erlang:finish_after_on_load/2 # +# New Bifs in R13B4 +# +bif erlang:binary_to_term/2 + +# # Obsolete # diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 5ea47e16f5..cc69977b79 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -703,6 +703,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fdprintf(fd, "System version: "); erts_print_system_version(fd, NULL, NULL); erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); + erts_fdprintf(fd, "Taints: "); + erts_print_nif_taints(fd, NULL); erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); info(fd, NULL); /* General system info */ if (process_tab != NULL) /* XXX true at init */ diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index b692832677..126ec7cc73 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -50,18 +50,16 @@ static ERTS_INLINE void maybe_shrink(Process* p, Eterm* hp, Eterm res, Uint allo if (is_immed(res)) { if (p->heap <= hp && hp < p->htop) { p->htop = hp; -#if defined(CHECK_FOR_HOLES) - } else { - erts_arith_shrink(p, hp); -#endif + } + else { + erts_heap_frag_shrink(p, hp); } } else if ((actual = bignum_header_arity(*hp)+1) < alloc) { if (p->heap <= hp && hp < p->htop) { p->htop = hp+actual; -#if defined(CHECK_FOR_HOLES) - } else { - erts_arith_shrink(p, hp+actual); -#endif + } + else { + erts_heap_frag_shrink(p, hp+actual); } } } @@ -397,12 +395,11 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) need_heap = BIG_NEED_SIZE(sz); hp = HAlloc(p, need_heap); res = big_plus(arg1, arg2, hp); + maybe_shrink(p, hp, res, need_heap); if (is_nil(res)) { - erts_arith_shrink(p, hp); p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } - maybe_shrink(p, hp, res, need_heap); return res; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): if (big_to_double(arg1, &f1.fd) < 0) { @@ -533,12 +530,11 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) need_heap = BIG_NEED_SIZE(sz); hp = HAlloc(p, need_heap); res = big_minus(arg1, arg2, hp); + maybe_shrink(p, hp, res, need_heap); if (is_nil(res)) { - erts_arith_shrink(p, hp); p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } - maybe_shrink(p, hp, res, need_heap); return res; default: goto badarith; @@ -731,12 +727,11 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) * the absolute value of the other is > 1. */ + maybe_shrink(p, hp, res, need_heap); if (is_nil(res)) { - erts_arith_shrink(p, hp); p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; - } - maybe_shrink(p, hp, res, need_heap); + } return res; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): if (big_to_double(arg1, &f1.fd) < 0) { @@ -956,12 +951,11 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2) need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i); hp = HAlloc(p, need); arg1 = big_div(arg1, arg2, hp); + maybe_shrink(p, hp, arg1, need); if (is_nil(arg1)) { - erts_arith_shrink(p, hp); p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } - maybe_shrink(p, hp, arg1, need); } return arg1; default: @@ -1004,12 +998,11 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) Eterm* hp = HAlloc(p, need); arg1 = big_rem(arg1, arg2, hp); + maybe_shrink(p, hp, arg1, need); if (is_nil(arg1)) { - erts_arith_shrink(p, hp); p->freason = SYSTEM_LIMIT; return THE_NON_VALUE; } - maybe_shrink(p, hp, arg1, need); } return arg1; default: @@ -1147,7 +1140,7 @@ trim_heap(Process* p, Eterm* hp, Eterm res) * a garbage collection if there is insufficient heap space. */ -#define erts_arith_shrink horrible error +#define erts_heap_frag_shrink horrible error #define maybe_shrink horrible error Eterm diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 8b47db10dd..440b0b4f14 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -318,6 +318,10 @@ double_to_integer(Process* p, double x) * The following code is used when a guard that may build on the * heap is called directly. They must not use HAlloc(), but must * do a garbage collection if there is insufficient heap space. + * + * Important note: All error checking MUST be done before doing + * a garbage collection. The compiler assumes that all registers + * are still valid if a guard BIF generates an exception. */ #define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need)) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 60216aa8e4..5ff1f794df 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -543,6 +543,8 @@ static Eterm pi_args[] = { am_last_calls, am_total_heap_size, am_suspending, + am_min_heap_size, + am_min_bin_vheap_size, #ifdef HYBRID am_message_binary #endif @@ -589,8 +591,10 @@ pi_arg2ix(Eterm arg) case am_last_calls: return 24; case am_total_heap_size: return 25; case am_suspending: return 26; + case am_min_heap_size: return 27; + case am_min_bin_vheap_size: return 28; #ifdef HYBRID - case am_message_binary: return 27; + case am_message_binary: return 29; #endif default: return -1; } @@ -1355,6 +1359,30 @@ process_info_aux(Process *BIF_P, break; } + case am_fullsweep_after: { + Uint hsz = 3; + (void) erts_bld_uint(NULL, &hsz, MAX_GEN_GCS(rp)); + hp = HAlloc(BIF_P, hsz); + res = erts_bld_uint(&hp, NULL, MAX_GEN_GCS(rp)); + break; + } + + case am_min_heap_size: { + Uint hsz = 3; + (void) erts_bld_uint(NULL, &hsz, MIN_HEAP_SIZE(rp)); + hp = HAlloc(BIF_P, hsz); + res = erts_bld_uint(&hp, NULL, MIN_HEAP_SIZE(rp)); + break; + } + + case am_min_bin_vheap_size: { + Uint hsz = 3; + (void) erts_bld_uint(NULL, &hsz, MIN_VHEAP_SIZE(rp)); + hp = HAlloc(BIF_P, hsz); + res = erts_bld_uint(&hp, NULL, MIN_VHEAP_SIZE(rp)); + break; + } + case am_total_heap_size: { ErlMessage *mp; Uint total_heap_size; @@ -1433,15 +1461,17 @@ process_info_aux(Process *BIF_P, DECL_AM(minor_gcs); Eterm t; - hp = HAlloc(BIF_P, 3+2+3+2+3); - t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); - hp += 3; - res = CONS(hp, t, NIL); - hp += 2; - t = TUPLE2(hp, am_fullsweep_after, make_small(MAX_GEN_GCS(rp))); - hp += 3; - res = CONS(hp, t, res); - hp += 2; + hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3); /* last "3" is for outside tuple */ + + t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); hp += 3; + res = CONS(hp, t, NIL); hp += 2; + t = TUPLE2(hp, am_fullsweep_after, make_small(MAX_GEN_GCS(rp))); hp += 3; + res = CONS(hp, t, res); hp += 2; + + t = TUPLE2(hp, am_min_heap_size, make_small(MIN_HEAP_SIZE(rp))); hp += 3; + res = CONS(hp, t, res); hp += 2; + t = TUPLE2(hp, am_min_bin_vheap_size, make_small(MIN_VHEAP_SIZE(rp))); hp += 3; + res = CONS(hp, t, res); hp += 2; break; } @@ -1897,16 +1927,32 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); - hp = HAlloc(BIF_P, 3+2); - res = TUPLE2(hp, am_fullsweep_after, make_small(val)); - hp += 3; - res = CONS(hp, res, NIL); + Eterm tup; + hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); + + tup = TUPLE2(hp, am_fullsweep_after, make_small(val)); hp += 3; + res = CONS(hp, tup, NIL); hp += 2; + + tup = TUPLE2(hp, am_min_heap_size, make_small(H_MIN_SIZE)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE2(hp, am_min_bin_vheap_size, make_small(BIN_VH_MIN_SIZE)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); + } else if (BIF_ARG_1 == am_min_heap_size) { + hp = HAlloc(BIF_P, 3); + res = TUPLE2(hp, am_min_heap_size,make_small(H_MIN_SIZE)); + BIF_RET(res); + } else if (BIF_ARG_1 == am_min_bin_vheap_size) { + hp = HAlloc(BIF_P, 3); + res = TUPLE2(hp, am_min_bin_vheap_size,make_small(BIN_VH_MIN_SIZE)); + BIF_RET(res); } else if (BIF_ARG_1 == am_process_count) { BIF_RET(make_small(erts_process_count())); } else if (BIF_ARG_1 == am_process_limit) { @@ -3131,6 +3177,13 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)) { BIF_RET(am_true); } + else if (ERTS_IS_ATOM_STR("force_heap_frags", BIF_ARG_1)) { +#ifdef FORCE_HEAP_FRAGS + BIF_RET(am_true); +#else + BIF_RET(am_false); +#endif + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 16abab65b0..c027cd5984 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -884,7 +884,7 @@ re_run_3(BIF_ALIST_3) int capture_count; if (pflags & PARSE_FLAG_UNICODE && - (!is_binary(BIF_ARG_1) || + (!is_binary(BIF_ARG_2) || !is_binary(BIF_ARG_1) || (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } @@ -1020,6 +1020,9 @@ re_run_3(BIF_ALIST_3) goto handle_iolist; } pb = (ProcBin *) bptr; + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } restart.subject = (char *) (pb->bytes+offset); restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index d3a916d2d9..b421da591b 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -2529,7 +2529,9 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key) this = this->right; } } - release_stack(tb,stack); + if (stack) { + release_stack(tb,stack); + } return this; } diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 34ce87bc5d..e5c3c76fdd 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -344,7 +344,7 @@ void erts_check_for_holes(Process* p) if (hf == p->last_mbuf) { break; } - check_memory(hf->mem, hf->mem+hf->size); + check_memory(hf->mem, hf->mem+hf->used_size); } p->last_mbuf = MBUF(p); } @@ -386,7 +386,7 @@ void erts_check_heap(Process *p) } while (bp) { - erts_check_memory(p,bp->mem,bp->mem + bp->size); + erts_check_memory(p,bp->mem,bp->mem + bp->used_size); bp = bp->next; } } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6945317e65..e9bf37a173 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifdef HAVE_CONFIG_H @@ -961,12 +961,13 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) n_htop = sweep_one_area(n_heap, n_htop, heap, heap_size); } else { Eterm* n_hp = n_heap; + Eterm* ptr; + Eterm val; + Eterm gval; while (n_hp != n_htop) { - Eterm* ptr; - Eterm val; - Eterm gval = *n_hp; - + ASSERT(n_hp < n_htop); + gval = *n_hp; switch (primary_tag(gval)) { case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); @@ -1402,68 +1403,6 @@ remove_message_buffers(Process* p) } } -/* - * Go through one root set array, move everything that it is one of the - * heap fragments to our new heap. - */ -static Eterm* -collect_root_array(Process* p, Eterm* n_htop, Eterm* objv, int nobj) -{ - ErlHeapFragment* qb; - Eterm gval; - Eterm* ptr; - Eterm val; - - ASSERT(p->htop != NULL); - while (nobj--) { - gval = *objv; - - switch (primary_tag(gval)) { - - case TAG_PRIMARY_BOXED: { - ptr = boxed_val(gval); - val = *ptr; - if (IS_MOVED(val)) { - ASSERT(is_boxed(val)); - *objv++ = val; - } else { - for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { - MOVE_BOXED(ptr,val,n_htop,objv); - break; - } - } - objv++; - } - break; - } - - case TAG_PRIMARY_LIST: { - ptr = list_val(gval); - val = *ptr; - if (is_non_value(val)) { - *objv++ = ptr[1]; - } else { - for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { - MOVE_CONS(ptr,val,n_htop,objv); - break; - } - } - objv++; - } - break; - } - - default: { - objv++; - break; - } - } - } - return n_htop; -} - #ifdef HARDDEBUG /* @@ -1707,11 +1646,13 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) { - while (n_hp != n_htop) { - Eterm* ptr; - Eterm val; - Eterm gval = *n_hp; + Eterm* ptr; + Eterm val; + Eterm gval; + while (n_hp != n_htop) { + ASSERT(n_hp < n_htop); + gval = *n_hp; switch (primary_tag(gval)) { case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); @@ -1820,6 +1761,35 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr } /* + * Move an area (heap fragment) by sweeping over it and set move markers. + */ +static Eterm* +move_one_area(Eterm* n_htop, char* src, Uint src_size) +{ + Eterm* ptr = (Eterm*) src; + Eterm* end = ptr + src_size/sizeof(Eterm); + Eterm dummy_ref; + + while (ptr != end) { + Eterm val; + ASSERT(ptr < end); + val = *ptr; + ASSERT(val != ERTS_HOLE_MARKER); + if (is_header(val)) { + ASSERT(ptr + header_arity(val) < end); + MOVE_BOXED(ptr, val, n_htop, &dummy_ref); + } + else { /* must be a cons cell */ + ASSERT(ptr+1 < end); + MOVE_CONS(ptr, val, n_htop, &dummy_ref); + ptr += 2; + } + } + + return n_htop; +} + +/* * Collect heap fragments and check that they point in the correct direction. */ @@ -1830,7 +1800,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, ErlHeapFragment* qb; char* frag_begin; Uint frag_size; - ErlMessage* mp; /* * We don't allow references to a heap fragments from the stack, heap, @@ -1845,65 +1814,44 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, #endif /* - * Go through the subset of the root set that is allowed to - * reference data in heap fragments and move data from heap fragments - * to our new heap. - */ - - if (nobj != 0) { - n_htop = collect_root_array(p, n_htop, objv, nobj); - } - if (is_not_immed(p->fvalue)) { - n_htop = collect_root_array(p, n_htop, &p->fvalue, 1); - } - if (is_not_immed(p->ftrace)) { - n_htop = collect_root_array(p, n_htop, &p->ftrace, 1); - } - if (is_not_immed(p->seq_trace_token)) { - n_htop = collect_root_array(p, n_htop, &p->seq_trace_token, 1); - } - if (is_not_immed(p->group_leader)) { - n_htop = collect_root_array(p, n_htop, &p->group_leader, 1); - } - - /* - * Go through the message queue, move everything that is in one of the - * heap fragments to our new heap. - */ - - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - /* - * In most cases, mp->data.attached points to a heap fragment which is - * self-contained and we will copy it to the heap at the - * end of the GC to avoid scanning it. - * - * In a few cases, however, such as in process_info(Pid, messages) - * and trace_delivered/1, a new message points to a term that has - * been allocated by HAlloc() and mp->data.attached is NULL. Therefore - * we need this loop. - */ - if (mp->data.attached == NULL) { - n_htop = collect_root_array(p, n_htop, mp->m, 2); - } - } - - /* - * Now all references in the root set point to the new heap. However, - * many references on the new heap point to heap fragments. - */ - + * Move the heap fragments to the new heap. Note that no GC is done on + * the heap fragments. Any garbage will thus be moved as well and survive + * until next GC. + */ qb = MBUF(p); - while (qb != NULL) { - frag_begin = (char *) qb->mem; - frag_size = qb->size * sizeof(Eterm); + while (qb != NULL) { + frag_size = qb->used_size * sizeof(Eterm); if (frag_size != 0) { - n_htop = sweep_one_area(n_hstart, n_htop, frag_begin, frag_size); + frag_begin = (char *) qb->mem; + n_htop = move_one_area(n_htop, frag_begin, frag_size); } qb = qb->next; } return n_htop; } +#ifdef DEBUG +static Eterm follow_moved(Eterm term) +{ + Eterm* ptr; + switch (primary_tag(term)) { + case TAG_PRIMARY_IMMED1: + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(term); + if (IS_MOVED(*ptr)) term = *ptr; + break; + case TAG_PRIMARY_LIST: + ptr = list_val(term); + if (is_non_value(ptr[0])) term = ptr[1]; + break; + default: + abort(); + } + return term; +} +#endif + static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { @@ -1932,7 +1880,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } ASSERT((is_nil(p->seq_trace_token) || - is_tuple(p->seq_trace_token) || + is_tuple(follow_moved(p->seq_trace_token)) || is_atom(p->seq_trace_token))); if (is_not_immed(p->seq_trace_token)) { roots[n].v = &p->seq_trace_token; @@ -1944,7 +1892,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) is_internal_pid(p->tracer_proc) || is_internal_port(p->tracer_proc)); - ASSERT(is_pid(p->group_leader)); + ASSERT(is_pid(follow_moved(p->group_leader))); if (is_not_immed(p->group_leader)) { roots[n].v = &p->group_leader; roots[n].sz = 1; @@ -2083,23 +2031,45 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) } static Uint -next_vheap_size(Uint vheap, Uint vheap_sz) { - if (vheap < H_MIN_SIZE) { - return H_MIN_SIZE; - } +do_next_vheap_size(Uint vheap, Uint vheap_sz) { + + /* grow + * + * vheap_sz ====================== + * + * vheap 75% + grow + * ---------------------- + * + * vheap 25 - 75% same + * ---------------------- + * + * vheap ~ - 25% shrink + * + * ---------------------- + */ + + if (vheap > (Uint) (vheap_sz*3/4)) { + + while(vheap > (Uint) (vheap_sz*3/4)) { + vheap_sz = vheap_sz*2; + } - /* grow */ - if (vheap > vheap_sz) { - return erts_next_heap_size(2*vheap, 0); + return erts_next_heap_size(vheap_sz, 0); } - /* shrink */ - if ( vheap < vheap_sz/2) { - return (Uint)vheap_sz*3/4; + + if (vheap < (Uint) (vheap_sz/4)) { + return erts_next_heap_size((Uint) (vheap_sz / 2), 0); } return vheap_sz; + } +static Uint +next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) { + vheap_sz = do_next_vheap_size(vheap, vheap_sz); + return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz; +} static void sweep_proc_externals(Process *p, int fullsweep) @@ -2302,8 +2272,8 @@ sweep_proc_bins(Process *p, int fullsweep) FLAGS(p) |= F_NEED_FULLSWEEP; } - BIN_VHEAP_SZ(p) = next_vheap_size(bin_vheap, BIN_VHEAP_SZ(p)); - BIN_OLD_VHEAP_SZ(p) = next_vheap_size(BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p)); + BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); + BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p)); MSO(p).overhead = bin_vheap; /* diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 8afd349b85..17cf3b9597 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -84,9 +84,10 @@ int erts_use_sender_punish; * Configurable parameters. */ -Uint display_items; /* no of items to display in traces etc */ +Uint display_items; /* no of items to display in traces etc */ Uint display_loads; /* print info about loaded modules */ int H_MIN_SIZE; /* The minimum heap grain */ +int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ Uint32 erts_debug_flags; /* Debug flags. */ #ifdef ERTS_OPCODE_COUNTER_SUPPORT @@ -118,6 +119,8 @@ int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is * not and/or it is too slow. */ +int erts_atom_table_size = ATOM_LIMIT; /* Maximum number of atoms */ + int erts_modified_timing_level; int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ @@ -252,7 +255,8 @@ erl_init(void) no_schedulers, no_schedulers_online); - H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); + H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); + BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); erts_init_trace(); erts_init_binary(); @@ -513,67 +517,73 @@ void erts_usage(void) /* erts_fprintf(stderr, "-# number set the number of items to be used in traces etc\n"); */ - erts_fprintf(stderr, "-a size suggested stack size in kilo words for threads\n"); - erts_fprintf(stderr, " in the async-thread pool, valid range is [%d-%d]\n", + erts_fprintf(stderr, "-a size suggested stack size in kilo words for threads\n"); + erts_fprintf(stderr, " in the async-thread pool, valid range is [%d-%d]\n", ERTS_ASYNC_THREAD_MIN_STACK_SIZE, ERTS_ASYNC_THREAD_MAX_STACK_SIZE); - erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); - erts_fprintf(stderr, " valid range is [0-%d]\n", + erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); + erts_fprintf(stderr, " valid range is [0-%d]\n", ERTS_MAX_NO_OF_ASYNC_THREADS); - erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); - erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); - erts_fprintf(stderr, " handler, i to ignore break signals\n"); + erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); + erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); + erts_fprintf(stderr, " handler, i to ignore break signals\n"); /* erts_fprintf(stderr, "-b func set the boot function (default boot)\n"); */ - erts_fprintf(stderr, "-c disable continuous date/time correction with\n"); - erts_fprintf(stderr, " respect to uptime\n"); + erts_fprintf(stderr, "-c disable continuous date/time correction with\n"); + erts_fprintf(stderr, " respect to uptime\n"); - erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); - erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); + erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); + erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); - erts_fprintf(stderr, "-h number set minimum heap size in words (default %d)\n", + erts_fprintf(stderr, "-hms size set minimum heap size in words (default %d)\n", H_DEFAULT_SIZE); + erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", + VH_DEFAULT_SIZE); /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ - erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); + erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); - erts_fprintf(stderr, "-l turn on auto load tracing\n"); + erts_fprintf(stderr, "-l turn on auto load tracing\n"); - erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); - erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); + erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); + erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); - erts_fprintf(stderr, "-P number set maximum number of processes on this node,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, "-P number set maximum number of processes on this node,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES); - erts_fprintf(stderr, "-R number set compatibility release number,\n"); - erts_fprintf(stderr, " valid range [%d-%d]\n", + erts_fprintf(stderr, "-R number set compatibility release number,\n"); + erts_fprintf(stderr, " valid range [%d-%d]\n", ERTS_MIN_COMPAT_REL, this_rel_num()); - erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); - erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); - erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); - erts_fprintf(stderr, "-sct cput set cpu topology,\n"); - erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); - erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); + erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); + erts_fprintf(stderr, "-sct cput set cpu topology,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE); - erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); - erts_fprintf(stderr, " schedulers online (n2), valid range for both\n"); - erts_fprintf(stderr, " numbers are [1-%d]\n", + erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); + erts_fprintf(stderr, " schedulers online (n2), valid range for both\n"); + erts_fprintf(stderr, " numbers are [1-%d]\n", ERTS_MAX_NO_OF_SCHEDULERS); - erts_fprintf(stderr, "-T number set modified timing level,\n"); - erts_fprintf(stderr, " valid range is [0-%d]\n", + erts_fprintf(stderr, "-t size set the maximum number of atoms the " + "emulator can handle\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", + MIN_ATOM_TABLE_SIZE, MAX_ATOM_TABLE_SIZE); + erts_fprintf(stderr, "-T number set modified timing level,\n"); + erts_fprintf(stderr, " valid range is [0-%d]\n", ERTS_MODIFIED_TIMING_LEVELS-1); - erts_fprintf(stderr, "-V print Erlang version\n"); + erts_fprintf(stderr, "-V print Erlang version\n"); - erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); + erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); - erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); - erts_fprintf(stderr, " see error_logger documentation for details\n"); + erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); + erts_fprintf(stderr, " see error_logger documentation for details\n"); erts_fprintf(stderr, "\n"); erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); @@ -604,6 +614,7 @@ early_init(int *argc, char **argv) /* erts_async_max_threads = 0; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; H_MIN_SIZE = H_DEFAULT_SIZE; + BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE; erts_initialized = 0; @@ -922,17 +933,40 @@ erl_start(int argc, char **argv) fprintf(stderr, "The undocumented +H option has been removed (R10B-6).\n\n"); break; - case 'h': - /* set default heap size */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if ((H_MIN_SIZE = atoi(arg)) <= 0) { - erts_fprintf(stderr, "bad heap size %s\n", arg); - erts_usage(); + case 'h': { + char *sub_param = argv[i]+2; + /* set default heap size + * + * h|ms - min_heap_size + * h|mbs - min_bin_vheap_size + * + */ + if (has_prefix("mbs", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if ((BIN_VH_MIN_SIZE = atoi(arg)) <= 0) { + erts_fprintf(stderr, "bad heap size %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using minimum binary virtual heap size %d\n", BIN_VH_MIN_SIZE)); + + } else if (has_prefix("ms", sub_param)) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if ((H_MIN_SIZE = atoi(arg)) <= 0) { + erts_fprintf(stderr, "bad heap size %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE)); + } else { + /* backward compatibility */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + if ((H_MIN_SIZE = atoi(arg)) <= 0) { + erts_fprintf(stderr, "bad heap size %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE)); } - VERBOSE(DEBUG_SYSTEM, - ("using minimum heap size %d\n",H_MIN_SIZE)); break; - + } case 'd': /* * Never produce crash dumps for internally detected @@ -1112,6 +1146,22 @@ erl_start(int argc, char **argv) } break; } + case 't': + /* set atom table size */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + errno = 0; + erts_atom_table_size = strtol(arg, NULL, 10); + if (errno != 0 || + erts_atom_table_size < MIN_ATOM_TABLE_SIZE || + erts_atom_table_size > MAX_ATOM_TABLE_SIZE) { + erts_fprintf(stderr, "bad atom table size %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("setting maximum number of atoms to %d\n", + erts_atom_table_size)); + break; + case 'T' : arg = get_arg(argv[i]+2, argv[i+1], &i); errno = 0; diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 25f1d420d1..074b08ea57 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -84,6 +84,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "reg_tab", NULL }, { "migration_info_update", NULL }, { "proc_main", "pid" }, +#ifdef HIPE + { "hipe_mfait_lock", NULL }, +#endif { "nodes_monitors", NULL }, { "driver_list", NULL }, { "proc_link", "pid" }, diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 81fbdfbd5a..a056fce0c5 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -114,12 +114,8 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - (sizeof(ErlHeapFragment) - - sizeof(Eterm) - + bp->size*sizeof(Eterm)), - (sizeof(ErlHeapFragment) - - sizeof(Eterm) - + size*sizeof(Eterm))); + ERTS_HEAP_FRAG_SIZE(bp->size), + ERTS_HEAP_FRAG_SIZE(size)); if (bp != nbp) { Uint off_sz = size < nbp->size ? size : nbp->size; Eterm *sp = &bp->mem[0]; @@ -140,7 +136,7 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif } nbp->size = size; - + nbp->used_size = size; #ifdef HARD_DEBUG for (i = 0; i < brefs_size; i++) @@ -175,9 +171,7 @@ free_message_buffer(ErlHeapFragment* bp) erts_cleanup_offheap(&bp->off_heap); ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - (sizeof(ErlHeapFragment) - - sizeof(Eterm) - + bp->size*sizeof(Eterm))); + ERTS_HEAP_FRAG_SIZE(bp->size)); } static ERTS_INLINE void diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index f14f14a586..5cf7c209bd 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -50,19 +50,10 @@ struct erl_heap_fragment { ErlHeapFragment* next; /* Next heap fragment */ ErlOffHeap off_heap; /* Offset heap data. */ unsigned size; /* Size in words of mem */ + unsigned used_size; /* With terms to be moved to heap by GC */ Eterm mem[1]; /* Data */ }; -#define ERTS_SET_MBUF_HEAP_END(BP, HENDP) \ -do { \ - unsigned real_size__ = (BP)->size; \ - ASSERT((BP)->mem <= (HENDP) && (HENDP) <= (BP)->mem + real_size__); \ - (BP)->size = (HENDP) - (BP)->mem; \ - /* We do not reallocate since buffer *might* be moved. */ \ - /* FIXME: Memory count is wrong, but at least it's almost */ \ - /* right... */ \ -} while (0) - typedef struct erl_mesg { struct erl_mesg* next; /* Next message */ union { @@ -196,10 +187,12 @@ do { \ #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) + #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ do { \ (HEAP_FRAG_P)->next = NULL; \ (HEAP_FRAG_P)->size = (DATA_WORDS); \ + (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ (HEAP_FRAG_P)->off_heap.mso = NULL; \ (HEAP_FRAG_P)->off_heap.funs = NULL; \ (HEAP_FRAG_P)->off_heap.externals = NULL; \ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index fa4454a3f3..2cb93112ae 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -35,13 +35,6 @@ #include <limits.h> -/* -static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need) -{ - return HAlloc(env->proc, need); -} -*/ - #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need); @@ -116,12 +109,21 @@ void enif_free(ErlNifEnv* env, void* ptr) erts_free(ERTS_ALC_T_NIF, ptr); } +int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_atom(term); +} int enif_is_binary(ErlNifEnv* env, ERL_NIF_TERM term) { return is_binary(term) && (binary_bitsize(term) % 8 == 0); } - + +int enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_ref(term); +} + int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) { @@ -157,6 +159,25 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) return 1; } +int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) +{ + Binary* oldbin; + Binary* newbin; + ASSERT(bin->ref_bin != NULL); + + oldbin = (Binary*) bin->ref_bin; + newbin = (Binary *) erts_bin_realloc_fnf(oldbin, size); + if (!newbin) { + return 0; + } + newbin->orig_size = size; + bin->ref_bin = newbin; + bin->data = (unsigned char*) newbin->orig_bytes; + bin->size = size; + return 1; +} + + void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) { if (bin->ref_bin == NULL) { @@ -177,6 +198,28 @@ void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) #endif } +int enif_is_identical(ErlNifEnv* env, Eterm lhs, Eterm rhs) +{ + return EQ(lhs,rhs); +} + +int enif_compare(ErlNifEnv* env, Eterm lhs, Eterm rhs) +{ + return cmp(lhs,rhs); +} + +int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, Eterm** array) +{ + Eterm* ptr; + if (is_not_tuple(tpl)) { + return 0; + } + ptr = tuple_val(tpl); + *arity = arityval(*ptr); + *array = ptr+1; + return 1; +} + Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { if (bin->ref_bin == NULL) { @@ -203,7 +246,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) } } -ERL_NIF_TERM enif_make_badarg(ErlNifEnv* env) +Eterm enif_make_badarg(ErlNifEnv* env) { BIF_ERROR(env->proc, BADARG); } @@ -212,7 +255,7 @@ ERL_NIF_TERM enif_make_badarg(ErlNifEnv* env) int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) { #if SIZEOF_INT == SIZEOF_VOID_P - return term_to_Sint(term, ip); + return term_to_Sint(term, (Sint*)ip); #elif SIZEOF_LONG == SIZEOF_VOID_P Sint i; if (!term_to_Sint(term, &i) || i < INT_MIN || i > INT_MAX) { @@ -234,6 +277,17 @@ int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip) #endif } +int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) +{ + FloatDef f; + if (is_not_float(term)) { + return 0; + } + GET_DOUBLE(term, f); + *dp = f.fd; + return 1; +} + int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) { Eterm* val; @@ -267,12 +321,24 @@ ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) } +ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) +{ + Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); + FloatDef f; + f.fd = d; + PUT_DOUBLE(f, hp); + return make_float(hp); +} ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) { return am_atom_put(name, sys_strlen(name)); } +int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom) +{ + return erts_atom_get(name, sys_strlen(name), atom); +} ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) { @@ -320,13 +386,43 @@ ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) ERL_NIF_TERM enif_make_string(ErlNifEnv* env, const char* string) { - Sint n = strlen(string); + Sint n = sys_strlen(string); Eterm* hp = alloc_heap(env,n*2); return erts_bld_string_n(&hp,NULL,string,n); } +ERL_NIF_TERM enif_make_ref(ErlNifEnv* env) +{ + Eterm* hp = alloc_heap(env, REF_THING_SIZE); + return erts_make_ref_in_buffer(hp); +} + +#if 0 /* To be continued... */ +typedef struct enif_handle_type_t +{ + struct enif_handle_type_t* next; + struct enif_handle_type_t* prev; + const char* name; + void (*dtor)(void* obj); + erts_smp_atomic_t ref_cnt; /* num of handles of this type */ +}ErlNifHandleType; + +ErlNifHandleType* +enif_create_handle_type(ErlNifEnv* env, const char* type_name, + void (*dtor)(void *)) +{ + +} + +ERL_NIF_TERM enif_make_handle(ErlNifEnv* env, ErlNifHandleType* htype, void* obj) +{ +} +int enif_get_handle(ErlNifEnv* env, ERL_NIF_TERM term, void** objp) +{ +} +#endif /*************************************************************************** ** load_nif/2 ** @@ -361,7 +457,7 @@ static void refresh_cached_nif_data(Eterm* mod_code, ErlNifFunc* func = &mod_nif->entry->funcs[i]; Uint* code_ptr; - erts_atom_get(func->name, strlen(func->name), &f_atom); + erts_atom_get(func->name, sys_strlen(func->name), &f_atom); code_ptr = *get_func_pp(mod_code, f_atom, func->arity); code_ptr[5+2] = (Uint) mod_nif->data; } @@ -411,6 +507,18 @@ Eterm erts_nif_taints(Process* p) return list; } +void erts_print_nif_taints(int to, void* to_arg) +{ + struct tainted_module_t* t; + const char* delim = ""; + for (t=first_tainted_module ; t!=NULL; t=t->next) { + const Atom* atom = atom_tab(atom_val(t->module_atom)); + erts_print(to,to_arg,"%s%.*s", delim, atom->len, atom->name); + delim = ","; + } + erts_print(to,to_arg,"\n"); +} + static Eterm load_nif_error(Process* p, const char* atom, const char* format, ...) { @@ -428,7 +536,8 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, .. for (;;) { Eterm txt = erts_bld_string_n(hpp, &sz, dsbufp->str, dsbufp->str_len); - ret = erts_bld_tuple(hpp, szp, 3, am_error, mkatom(atom), txt); + ret = erts_bld_tuple(hpp, szp, 2, am_error, + erts_bld_tuple(hpp, szp, 2, mkatom(atom), txt)); if (hpp != NULL) { break; } @@ -489,8 +598,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) "module '%T' not allowed", mod_atom); } else if ((err=erts_sys_ddll_open2(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { - ret = load_nif_error(BIF_P, "load_failed", "Failed to load NIF library" - " %s: '%s'", lib_name, errdesc.str); + const char slogan[] = "Failed to load NIF library"; + if (strstr(errdesc.str, lib_name) != NULL) { + ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); + } + else { + ret = load_nif_error(BIF_P, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str); + } } else if (erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) { ret = load_nif_error(BIF_P, bad_lib, "Failed to find library init" @@ -517,12 +631,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { Uint** code_pp; ErlNifFunc* f = &entry->funcs[i]; - if (f->arity > 3) { - ret = load_nif_error(BIF_P,bad_lib,"Function arity too high for NIF %s/%u", - f->name, f->arity); - } - else if (!erts_atom_get(f->name, strlen(f->name), &f_atom) - || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { + if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom) + || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -612,7 +722,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) for (i=0; i < entry->num_of_funcs; i++) { Uint* code_ptr; - erts_atom_get(entry->funcs[i].name, strlen(entry->funcs[i].name), &f_atom); + erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); if (code_ptr[1] == 0) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 8650b7ce47..e5e6d65c0e 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -20,23 +20,27 @@ /* Include file for writers of Native Implemented Functions. */ -#define ERL_NIF_MAJOR_VERSION 0 -#define ERL_NIF_MINOR_VERSION 1 +/* Version history: +** 0.1: R13B03 +** 1.0: R13B04 +*/ +#define ERL_NIF_MAJOR_VERSION 1 +#define ERL_NIF_MINOR_VERSION 0 #include <stdlib.h> typedef unsigned long ERL_NIF_TERM; +struct enif_environment_t; +typedef struct enif_environment_t ErlNifEnv; + typedef struct { const char* name; unsigned arity; - void* fptr; //ERL_NIF_TERM (*fptr)(void*, ...); + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); }ErlNifFunc; -struct enif_environment_t; -typedef struct enif_environment_t ErlNifEnv; - typedef struct enif_entry_t { int major; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 400c1822cc..7556806ce4 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -25,44 +25,68 @@ ERL_NIF_API_FUNC_DECL(void*,enif_get_data,(ErlNifEnv*)); ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(ErlNifEnv*, size_t size)); ERL_NIF_API_FUNC_DECL(void,enif_free,(ErlNifEnv*, void* ptr)); +ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(ErlNifEnv*, unsigned size, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifEnv*, ErlNifBinary* bin, unsigned size)); ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifEnv*, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp)); ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)); +ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, ERL_NIF_TERM** array)); +ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_compare,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name)); +ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string,(ErlNifEnv* env, const char* string)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ref,(ErlNifEnv* env)); +/* +** Add last to keep compatibility on Windows!!! +*/ #endif #ifdef ERL_NIF_API_FUNC_MACRO # define enif_get_data ERL_NIF_API_FUNC_MACRO(enif_get_data) # define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc) # define enif_free ERL_NIF_API_FUNC_MACRO(enif_free) +# define enif_is_atom ERL_NIF_API_FUNC_MACRO(enif_is_atom) # define enif_is_binary ERL_NIF_API_FUNC_MACRO(enif_is_binary) +# define enif_is_ref ERL_NIF_API_FUNC_MACRO(enif_is_ref) # define enif_inspect_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_binary) # define enif_alloc_binary ERL_NIF_API_FUNC_MACRO(enif_alloc_binary) +# define enif_realloc_binary ERL_NIF_API_FUNC_MACRO(enif_realloc_binary) # define enif_release_binary ERL_NIF_API_FUNC_MACRO(enif_release_binary) # define enif_get_int ERL_NIF_API_FUNC_MACRO(enif_get_int) # define enif_get_ulong ERL_NIF_API_FUNC_MACRO(enif_get_ulong) +# define enif_get_double ERL_NIF_API_FUNC_MACRO(enif_get_double) +# define enif_get_tuple ERL_NIF_API_FUNC_MACRO(enif_get_tuple) # define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell) +# define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) +# define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) # define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) # define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) # define enif_make_int ERL_NIF_API_FUNC_MACRO(enif_make_int) # define enif_make_ulong ERL_NIF_API_FUNC_MACRO(enif_make_ulong) +# define enif_make_double ERL_NIF_API_FUNC_MACRO(enif_make_double) # define enif_make_atom ERL_NIF_API_FUNC_MACRO(enif_make_atom) +# define enif_make_existing_atom ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom) # define enif_make_tuple ERL_NIF_API_FUNC_MACRO(enif_make_tuple) # define enif_make_list ERL_NIF_API_FUNC_MACRO(enif_make_list) # define enif_make_list_cell ERL_NIF_API_FUNC_MACRO(enif_make_list_cell) # define enif_make_string ERL_NIF_API_FUNC_MACRO(enif_make_string) +# define enif_make_ref ERL_NIF_API_FUNC_MACRO(enif_make_ref) + #endif diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 0fb264a53c..0b6bb0d8e9 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -575,7 +575,7 @@ erts_port_task_schedule(Eterm id, } #endif - ASSERT(!(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + ASSERT(!enq_port || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); ASSERT(pp->sched.taskq); ASSERT(ptp); @@ -601,6 +601,15 @@ erts_port_task_schedule(Eterm id, break; } +#ifndef ERTS_SMP + /* + * When (!enq_port && !pp->sched.exe_taskq) is true in the smp case, + * the port might not be in the run queue. If this is the case, another + * thread is in the process of enqueueing the port. This very seldom + * occur, but do occur and is a valid scenario. Debug info showing this + * enqueue in progress must be introduced before we can enable (modified + * versions of these) assertions in the smp case again. + */ #if defined(HARD_DEBUG) if (pp->sched.exe_taskq || enq_port) ERTS_PT_CHK_NOT_IN_PORTQ(runq, pp); @@ -612,6 +621,7 @@ erts_port_task_schedule(Eterm id, ASSERT(pp->sched.prev || runq->ports.start == pp); } #endif +#endif if (!enq_port) { ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -902,25 +912,45 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) *curr_port_pp = NULL; - if (pp->sched.taskq) { +#ifdef ERTS_SMP + ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read(&pp->run_queue)); +#endif + + if (!pp->sched.taskq) { + ASSERT(pp->sched.exe_taskq); + pp->sched.exe_taskq = NULL; + } + else { +#ifdef ERTS_SMP + ErtsRunQueue *xrunq; +#endif + ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); ASSERT(pp->sched.taskq->first); - enqueue_port(runq, pp); - port_was_enqueued = 1; - - /* - erts_smp_notify_inc_runq(); - * No need to notify schedulers about the increase in run - * queue length since at least this thread, which is a - * scheduler, will discover that the port run queue isn't - * empty before trying to go to sleep. - */ +#ifdef ERTS_SMP + xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); + if (!xrunq) { +#endif + enqueue_port(runq, pp); + ASSERT(pp->sched.exe_taskq); + pp->sched.exe_taskq = NULL; + /* No need to notify ourselves about inc in runq. */ +#ifdef ERTS_SMP + } + else { + /* Port emigrated ... */ + erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + enqueue_port(xrunq, pp); + ASSERT(pp->sched.exe_taskq); + pp->sched.exe_taskq = NULL; + erts_smp_notify_inc_runq(xrunq); + erts_smp_runq_unlock(xrunq); + } +#endif + port_was_enqueued = 1; } - ASSERT(pp->sched.exe_taskq); - pp->sched.exe_taskq = NULL; - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; ERTS_PT_CHK_PRES_PORTQ(runq, pp); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 9960172366..2789691c55 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -219,7 +219,7 @@ ErtsSchedulerData *erts_scheduler_data; ErtsAlignedRunQueue *erts_aligned_run_queues; Uint erts_no_run_queues; -typedef struct { +typedef union { ErtsSchedulerData esd; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; } ErtsAlignedSchedulerData; @@ -2522,8 +2522,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) NULL); } - erts_smp_atomic_inc(&schdlr_sspnd.active); - + active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); + if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED + && schdlr_sspnd.online == active_schedulers) { + schdlr_sspnd.changing = 0; + } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); if (erts_system_profile_flags.scheduler) @@ -2750,18 +2753,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (on) { - if (schdlr_sspnd.changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ - } - else if (erts_is_multi_scheduling_blocked()) { + + if (schdlr_sspnd.changing) { + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ + } + else if (on) { /* ------ BLOCK ------ */ + if (erts_is_multi_scheduling_blocked()) { plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; p->flags |= F_HAVE_BLCKD_MSCHED; ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); - res = 1; + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } else { p->flags |= F_HAVE_BLCKD_MSCHED; @@ -2842,10 +2846,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } } else if (!ongoing_multi_scheduling_block()) { + /* unblock not ongoing */ ASSERT(!schdlr_sspnd.msb.procs); res = ERTS_SCHDLR_SSPND_DONE; } - else { + else { /* ------ UNBLOCK ------ */ if (p->flags & F_HAVE_BLCKD_MSCHED) { ErtsProcList **plpp = &schdlr_sspnd.msb.procs; plp = schdlr_sspnd.msb.procs; @@ -2891,12 +2896,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) #endif p->flags &= ~F_HAVE_BLCKD_MSCHED; erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0); - if (schdlr_sspnd.online == 1) - /* No schedulers to resume */; + if (schdlr_sspnd.online == 1) { + /* No schedulers to resume */ + ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + schdlr_sspnd.changing = 0; + } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); wake_all_schedulers(); + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } else { int online = schdlr_sspnd.online; @@ -2928,9 +2937,8 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - schdlr_sspnd.changing = 0; res = ERTS_SCHDLR_SSPND_DONE; } } @@ -5517,6 +5525,8 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, p->run_queue = to_rq; enqueue_process(to_rq, p); + smp_notify_inc_runq(to_rq); + return ERTS_MIGRATE_SUCCESS; } #endif /* ERTS_SMP */ @@ -6272,7 +6282,7 @@ Process *schedule(Process *p, int calls) erts_check_my_tracer_proc(p); #endif - if ((FLAGS(p) & F_FORCE_GC) || (MSO(p).overhead >= BIN_VHEAP_SZ(p))) { + if ((FLAGS(p) & F_FORCE_GC) || (MSO(p).overhead > BIN_VHEAP_SZ(p))) { reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); if (reds < 0) { reds = 1; @@ -6683,13 +6693,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * noone except us has access to the process. */ if (so->flags & SPO_USE_ARGS) { - p->min_heap_size = so->min_heap_size; - p->prio = so->priority; - p->max_gen_gcs = so->max_gen_gcs; + p->min_heap_size = so->min_heap_size; + p->min_vheap_size = so->min_vheap_size; + p->prio = so->priority; + p->max_gen_gcs = so->max_gen_gcs; } else { - p->min_heap_size = H_MIN_SIZE; - p->prio = PRIORITY_NORMAL; - p->max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + p->min_heap_size = H_MIN_SIZE; + p->min_vheap_size = BIN_VH_MIN_SIZE; + p->prio = PRIORITY_NORMAL; + p->max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); } p->skipped = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -6736,9 +6748,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->heap_sz = sz; p->catches = 0; - p->bin_vheap_sz = H_MIN_SIZE; - p->bin_old_vheap_sz = H_MIN_SIZE; - p->bin_old_vheap = 0; + p->bin_vheap_sz = p->min_vheap_size; + p->bin_old_vheap_sz = p->min_vheap_size; + p->bin_old_vheap = 0; /* No need to initialize p->fcalls. */ @@ -6969,6 +6981,7 @@ void erts_init_empty_process(Process *p) p->gen_gcs = 0; p->max_gen_gcs = 0; p->min_heap_size = 0; + p->min_vheap_size = 0; p->status = P_RUNABLE; p->gcstatus = P_RUNABLE; p->rstatus = P_RUNABLE; @@ -6985,8 +6998,8 @@ void erts_init_empty_process(Process *p) p->ftrace = NIL; p->fcalls = 0; - p->bin_vheap_sz=H_MIN_SIZE; - p->bin_old_vheap_sz=H_MIN_SIZE; + p->bin_vheap_sz = BIN_VH_MIN_SIZE; + p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; #ifdef ERTS_SMP p->u.ptimer = NULL; @@ -8024,11 +8037,6 @@ erts_do_exit_process(Process* p, Eterm reason) if (p->bif_timers) erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); -#ifdef ERTS_SMP - if (p->flags & F_HAVE_BLCKD_MSCHED) - erts_block_multi_scheduling(p, ERTS_PROC_LOCKS_ALL, 0, 1); -#endif - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #ifdef ERTS_SMP @@ -8073,6 +8081,27 @@ continue_exit_process(Process *p erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #endif +#ifdef ERTS_SMP + if (p->flags & F_HAVE_BLCKD_MSCHED) { + ErtsSchedSuspendResult ssr; + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1); + switch (ssr) { + case ERTS_SCHDLR_SSPND_YIELD_RESTART: + goto yield; + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE: + case ERTS_SCHDLR_SSPND_YIELD_DONE: + p->flags &= ~F_HAVE_BLCKD_MSCHED; + break; + case ERTS_SCHDLR_SSPND_EINVAL: + default: + erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + __FILE__, __LINE__, (int) ssr); + } + } +#endif + if (p->flags & F_USING_DB) { if (erts_db_process_exiting(p, ERTS_PROC_LOCK_MAIN)) goto yield; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7bae1e4efc..f58b6932b3 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -339,9 +339,14 @@ do { \ struct ErtsSchedulerData_ { #ifdef ERTS_SMP - ethr_tid tid; /* Thread id */ + /* + * Keep X registers first (so we get as many low + * numbered registers as possible in the same cache + * line). + */ Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */ FloatDef freg[MAX_REG]; /* Floating point registers. */ + ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ Process *free_process; @@ -472,6 +477,7 @@ struct ErtsPendingSuspend_ { # define MSO(p) (p)->off_heap # define MIN_HEAP_SIZE(p) (p)->min_heap_size +# define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -490,6 +496,7 @@ struct process { Eterm* hend; /* Heap end */ Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ + Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ #if !defined(NO_FPE_SIGNALS) volatile unsigned long fp_exception; @@ -649,6 +656,11 @@ struct process { * heap fragments. */ #endif + +#ifdef FORCE_HEAP_FRAGS + Uint space_verified; /* Avoid HAlloc forcing heap fragments when */ + Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */ +#endif }; #ifdef CHECK_FOR_HOLES @@ -720,8 +732,8 @@ typedef struct { * The following items are only initialized if the SPO_USE_ARGS flag is set. */ Uint min_heap_size; /* Minimum heap size (must be a valued returned - * from next_heap_size()). - */ + * from next_heap_size()). */ + Uint min_vheap_size; /* Minimum virtual heap size */ int priority; /* Priority for process. */ Uint16 max_gen_gcs; /* Maximum number of gen GCs before fullsweep. */ int scheduler; @@ -733,7 +745,20 @@ typedef struct { #define KILL_CATCHES(p) (p)->catches = -1 -void erts_arith_shrink(Process* p, Eterm* hp); +/* Shrink heap fragment from _last_ HAlloc. +*/ +ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) +{ + ErlHeapFragment* hf = MBUF(p); + + ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->size)); + + hf->used_size = hp - hf->mem; +} +#endif /* inline */ + Eterm* erts_heap_alloc(Process* p, Uint need); #ifdef CHECK_FOR_HOLES Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 76bfdecd9f..c15f85f8f1 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -754,12 +754,8 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) { SysTimeval now; - erts_smp_mtx_lock(&erts_timeofday_mtx); - sys_gettimeofday(&now); - erts_smp_mtx_unlock(&erts_timeofday_mtx); - *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 2afb16fc52..2842c2361a 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -2171,6 +2171,11 @@ erts_bif_trace(int bif_index, Process* p, void trace_gc(Process *p, Eterm what) { + ERTS_DECL_AM(bin_vheap_size); + ERTS_DECL_AM(bin_vheap_block_size); + ERTS_DECL_AM(bin_old_vheap_size); + ERTS_DECL_AM(bin_old_vheap_block_size); + ErlHeapFragment *bp = NULL; ErlOffHeap *off_heap; ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; /* Initialized @@ -2180,6 +2185,7 @@ trace_gc(Process *p, Eterm what) Eterm* hp; Eterm msg = NIL; Uint size; + Eterm tags[] = { am_old_heap_block_size, am_heap_block_size, @@ -2187,8 +2193,13 @@ trace_gc(Process *p, Eterm what) am_recent_size, am_stack_size, am_old_heap_size, - am_heap_size + am_heap_size, + AM_bin_vheap_size, + AM_bin_vheap_block_size, + AM_bin_old_vheap_size, + AM_bin_old_vheap_block_size }; + Uint values[] = { OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), @@ -2196,7 +2207,11 @@ trace_gc(Process *p, Eterm what) HIGH_WATER(p) - HEAP_START(p), STACK_START(p) - p->stop, OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0, - HEAP_TOP(p) - HEAP_START(p) + HEAP_TOP(p) - HEAP_START(p), + MSO(p).overhead, + BIN_VHEAP_SZ(p), + BIN_OLD_VHEAP(p), + BIN_OLD_VHEAP_SZ(p) }; Eterm local_heap[(sizeof(values)/sizeof(Uint)) *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 4d8315ab95..50b3e5b61c 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -22,6 +22,13 @@ /* #define ERTS_OPCODE_COUNTER_SUPPORT */ +/* FORCE_HEAP_FRAGS: + * Debug provocation to make HAlloc always create heap fragments (if allowed) + * even if there is room on heap. + */ +/* #define FORCE_HEAP_FRAGS */ + + #if defined(HYBRID) /* # define CHECK_FOR_HOLES */ #endif @@ -50,7 +57,8 @@ #define INPUT_REDUCTIONS (2 * CONTEXT_REDS) -#define H_DEFAULT_SIZE 233 /* default (heap + stack) min size */ +#define H_DEFAULT_SIZE 233 /* default (heap + stack) min size */ +#define VH_DEFAULT_SIZE 32768 /* default virtual (bin) heap min size (words) */ #ifdef HYBRID # define SH_DEFAULT_SIZE 2629425 /* default message area min size */ @@ -70,6 +78,7 @@ && (P) == (P)->scheduler_data->match_pseudo_process) \ || erts_is_system_blocked(0)) + #ifdef DEBUG /* * Debug HAlloc that initialize all memory to bad things. @@ -80,55 +89,43 @@ VERBOSE(DEBUG_ALLOCATION,("HAlloc @ 0x%08lx (%d) %s:%d\n", \ (unsigned long)HEAP_TOP(p),(sz),__FILE__,__LINE__)), \ */ -#ifdef CHECK_FOR_HOLES -#define HAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - ErtsHAllocLockCheck(p), \ - ((((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ - ? erts_heap_alloc((p),(sz)) \ - : (erts_set_hole_marker(HEAP_TOP(p), (sz)), \ - HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) +# ifdef CHECK_FOR_HOLES +# define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz)) +# else +# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),DEBUG_BAD_BYTE,(sz)*sizeof(Eterm*)) +# endif #else -#define HAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - ErtsHAllocLockCheck(p), \ - ((((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ - ? erts_heap_alloc((p),(sz)) \ - : (memset(HEAP_TOP(p),DEBUG_BAD_BYTE,(sz)*sizeof(Eterm*)), \ - HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) -#endif +# define INIT_HEAP_MEM(p,sz) ((void)0) +#endif /* DEBUG */ + + +#ifdef FORCE_HEAP_FRAGS +# define IS_FORCE_HEAP_FRAGS 1 #else +# define IS_FORCE_HEAP_FRAGS 0 +#endif /* * Allocate heap memory, first on the ordinary heap; * failing that, in a heap fragment. */ -#define HAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - ErtsHAllocLockCheck(p), \ - ((((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ - ? erts_heap_alloc((p),(sz)) \ - : (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) +#define HAlloc(p, sz) \ + (ASSERT_EXPR((sz) >= 0), \ + ErtsHAllocLockCheck(p), \ + (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ + ? erts_heap_alloc((p),(sz)) \ + : (INIT_HEAP_MEM(p,sz), \ + HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) -#endif /* DEBUG */ -#if defined(CHECK_FOR_HOLES) -# define HRelease(p, endp, ptr) \ +#define HRelease(p, endp, ptr) \ if ((ptr) == (endp)) { \ ; \ } else if (HEAP_START(p) <= (ptr) && (ptr) < HEAP_TOP(p)) { \ HEAP_TOP(p) = (ptr); \ } else { \ - erts_arith_shrink(p, ptr); \ + erts_heap_frag_shrink(p, ptr); \ } -#else -# define HRelease(p, endp, ptr) \ - if ((ptr) == (endp)) { \ - ; \ - } else if (HEAP_START(p) <= (ptr) && (ptr) < HEAP_TOP(p)) { \ - HEAP_TOP(p) = (ptr); \ - } -#endif #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) @@ -182,6 +179,9 @@ extern int num_instructions; /* Number of instruction in opc[]. */ #define MAX_PORT_LINK 8 /* Maximum number of links to a port */ extern int H_MIN_SIZE; /* minimum (heap + stack) */ +extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ + +extern int erts_atom_table_size;/* Atom table size */ #define ORIG_CREATION 0 diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index f856cce18f..b011d4c0de 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -271,11 +271,8 @@ erts_encode_ext_dist_header_size(ErtsAtomCacheMap *acmp) byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp) { -#ifndef ARCH_32 -#if ATOM_LIMIT >= (1UL << 32) -#error "ATOM_LIMIT too large for interal atom cache update instructions. New instructions needed." -#endif -#endif + /* Maximum number of atom must be less than the maximum of a 32 bits + unsigned integer. Check is done in erl_init.c, erl_start function. */ if (!acmp) return ctl_ext; else { @@ -1013,6 +1010,34 @@ term_to_binary_2(Process* p, Eterm Term, Eterm Flags) return erts_term_to_binary(p, Term, level, flags); } +static uLongf binary2term_uncomp_size(byte* data, Sint size) +{ + z_stream stream; + int err; + const uInt chunk_size = 64*1024; /* Ask tmp-alloc about a suitable size? */ + void* tmp_buf = erts_alloc(ERTS_ALC_T_TMP, chunk_size); + uLongf uncomp_size = 0; + + stream.next_in = (Bytef*)data; + stream.avail_in = (uInt)size; + stream.next_out = tmp_buf; + stream.avail_out = (uInt)chunk_size; + + erl_zlib_alloc_init(&stream); + + err = inflateInit(&stream); + if (err == Z_OK) { + while ((err = inflate(&stream, Z_NO_FLUSH)) == Z_OK) { + uncomp_size += chunk_size - stream.avail_out; + stream.next_out = tmp_buf; + stream.avail_out = chunk_size; + } + inflateEnd(&stream); + } + erts_free(ERTS_ALC_T_TMP, tmp_buf); + return err == Z_STREAM_END ? uncomp_size : 0; +} + static ERTS_INLINE Sint binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) { @@ -1036,10 +1061,18 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) state->extp = bytes; } else { - uLongf dest_len = get_int32(bytes+1); - state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len); + uLongf dest_len = (Uint32) get_int32(bytes+1); + bytes += 5; + size -= 5; + if (dest_len > 32*1024*1024 + || (state->extp = erts_alloc_fnf(ERTS_ALC_T_TMP, dest_len)) == NULL) { + if (dest_len != binary2term_uncomp_size(bytes, size)) { + goto error; + } + state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len); + } state->exttmp = 1; - if (erl_zlib_uncompress(state->extp, &dest_len, bytes+5, size-5) != Z_OK) + if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK) goto error; size = (Sint) dest_len; } @@ -1059,10 +1092,10 @@ binary2term_abort(ErtsBinary2TermState *state) } static ERTS_INLINE Eterm -binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) { Eterm res; - if (!dec_term(NULL, hpp, state->extp, ohp, &res)) + if (!dec_term(edep, hpp, state->extp, ohp, &res)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; @@ -1086,7 +1119,7 @@ erts_binary2term_abort(ErtsBinary2TermState *state) Eterm erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) { - return binary2term_create(state, hpp, ohp); + return binary2term_create(NULL,state, hpp, ohp); } BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) @@ -1114,7 +1147,67 @@ BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) hp = HAlloc(BIF_P, heap_size); endp = hp + heap_size; - res = binary2term_create(&b2ts, &hp, &MSO(BIF_P)); + res = binary2term_create(NULL, &b2ts, &hp, &MSO(BIF_P)); + + erts_free_aligned_binary_bytes(temp_alloc); + + if (hp > endp) { + erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", + __FILE__, __LINE__, hp-endp); + } + + HRelease(BIF_P, endp, hp); + + if (res == THE_NON_VALUE) + goto error; + + return res; +} + +BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) +{ + Sint heap_size; + Eterm res; + Eterm opts; + Eterm opt; + Eterm* hp; + Eterm* endp; + Sint size; + byte* bytes; + byte* temp_alloc = NULL; + ErtsBinary2TermState b2ts; + ErtsDistExternal fakedep; + + fakedep.flags = 0; + opts = BIF_ARG_2; + while (is_list(opts)) { + opt = CAR(list_val(opts)); + if (opt == am_safe) { + fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE; + } else { + goto error; + } + opts = CDR(list_val(opts)); + } + + if (is_not_nil(opts)) + goto error; + + if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) { + error: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P, BADARG); + } + size = binary_size(BIF_ARG_1); + + heap_size = binary2term_prepare(&b2ts, bytes, size); + if (heap_size < 0) + goto error; + + hp = HAlloc(BIF_P, heap_size); + endp = hp + heap_size; + + res = binary2term_create(&fakedep, &b2ts, &hp, &MSO(BIF_P)); erts_free_aligned_binary_bytes(temp_alloc); @@ -1300,7 +1393,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) switch (*ep++) { case ATOM_CACHE_REF: - if (!(edep->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB)) + if (!(edep && (edep->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB))) goto error; n = get_int8(ep); ep++; @@ -1312,13 +1405,18 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) case ATOM_EXT: len = get_int16(ep), ep += 2; - *objp = am_atom_put((char*)ep, len); - ep += len; - break; + goto dec_atom_common; case SMALL_ATOM_EXT: len = get_int8(ep); ep++; - *objp = am_atom_put((char*)ep, len); + dec_atom_common: + if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { + if (!erts_atom_get((char*)ep, len, objp)) { + goto error; + } + } else { + *objp = am_atom_put((char*)ep, len); + } ep += len; break; default: @@ -1775,9 +1873,80 @@ is_external_string(Eterm list, int* p_is_string) return len; } +/* Assumes that the ones to undo are preluding the lists. */ +static void +undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) +{ + const Uint area_sz = (end - start) * sizeof(Eterm); + struct proc_bin* mso; + struct proc_bin** mso_nextp = NULL; +#ifndef HYBRID /* FIND ME! */ + struct erl_fun_thing* funs; + struct erl_fun_thing** funs_nextp = NULL; +#endif + struct external_thing_* ext; + struct external_thing_** ext_nextp = NULL; + + for (mso = off_heap->mso; ; mso=mso->next) { + if (!in_area(mso, start, area_sz)) { + if (mso_nextp != NULL) { + *mso_nextp = NULL; + erts_cleanup_mso(off_heap->mso); + off_heap->mso = mso; + } + break; + } + mso_nextp = &mso->next; + } + +#ifndef HYBRID /* FIND ME! */ + for (funs = off_heap->funs; ; funs=funs->next) { + if (!in_area(funs, start, area_sz)) { + if (funs_nextp != NULL) { + *funs_nextp = NULL; + erts_cleanup_funs(off_heap->funs); + off_heap->funs = funs; + } + break; + } + funs_nextp = &funs->next; + } +#endif + for (ext = off_heap->externals; ; ext=ext->next) { + if (!in_area(ext, start, area_sz)) { + if (ext_nextp != NULL) { + *ext_nextp = NULL; + erts_cleanup_externals(off_heap->externals); + off_heap->externals = ext; + } + break; + } + ext_nextp = &ext->next; + } + + /* Assert that the ones to undo were indeed preluding the lists. */ +#ifdef DEBUG + for (mso = off_heap->mso; mso != NULL; mso=mso->next) { + ASSERT(!in_area(mso, start, area_sz)); + } +# ifndef HYBRID /* FIND ME! */ + for (funs = off_heap->funs; funs != NULL; funs=funs->next) { + ASSERT(!in_area(funs, start, area_sz)); + } +# endif + for (ext = off_heap->externals; ext != NULL; ext=ext->next) { + ASSERT(!in_area(ext, start, area_sz)); + } +#endif /* DEBUG */ +} + +/* Decode term from external format into *objp. +** On failure return NULL and (R13B04) *hpp will be unchanged. +*/ static byte* dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) { + Eterm* hp_saved = *hpp; int n; register Eterm* hp = *hpp; /* Please don't take the address of hp */ Eterm* next = objp; @@ -1864,13 +2033,18 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et case ATOM_EXT: n = get_int16(ep); ep += 2; - *objp = am_atom_put((char*)ep, n); - ep += n; - break; + goto dec_term_atom_common; case SMALL_ATOM_EXT: n = get_int8(ep); ep++; - *objp = am_atom_put((char*)ep, n); +dec_term_atom_common: + if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { + if (!erts_atom_get((char*)ep, n, objp)) { + goto error; + } + } else { + *objp = am_atom_put((char*)ep, n); + } ep += n; break; case LARGE_TUPLE_EXT: @@ -1973,7 +2147,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et ep = dec_pid(edep, hpp, ep, off_heap, objp); hp = *hpp; if (ep == NULL) { - return NULL; + goto error; } break; case PORT_EXT: @@ -2039,7 +2213,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et goto ref_ext_common; case NEW_REFERENCE_EXT: - ref_words = get_int16(ep); ep += 2; @@ -2209,7 +2382,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et ep = dec_term(edep, hpp, ep, off_heap, &temp); hp = *hpp; if (ep == NULL) { - return NULL; + goto error; } if (!is_small(temp)) { goto error; @@ -2218,6 +2391,10 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et if (arity < 0) { goto error; } + if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { + if (!erts_find_export_entry(mod, name, arity)) + goto error; + } *objp = make_export(hp); *hp++ = HEADER_EXPORT; *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); @@ -2235,8 +2412,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et Sint old_index; unsigned num_free; int i; - Eterm* temp_hp; - Eterm** hpp = &temp_hp; Eterm temp; ep += 4; /* Skip total size in bytes */ @@ -2248,23 +2423,16 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et num_free = get_int32(ep); ep += 4; hp += ERL_FUN_SIZE; - if (num_free > 0) { - /* Don't leave a hole in case we fail */ - *hp = make_pos_bignum_header(num_free-1); - } hp += num_free; - *hpp = hp; funp->thing_word = HEADER_FUN; funp->num_free = num_free; - funp->creator = NIL; /* Don't leave a hole in case we fail */ *objp = make_fun(funp); /* Module */ - if ((ep = dec_atom(edep, ep, &temp)) == NULL) { + if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - module = temp; - + *hpp = hp; /* Index */ if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { goto error; @@ -2321,17 +2489,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et Sint old_index; unsigned num_free; int i; - Eterm* temp_hp; - Eterm** hpp = &temp_hp; Eterm temp; num_free = get_int32(ep); ep += 4; hp += ERL_FUN_SIZE; - if (num_free > 0) { - /* Don't leave a hole in the heap in case we fail. */ - *hp = make_pos_bignum_header(num_free-1); - } hp += num_free; *hpp = hp; funp->thing_word = HEADER_FUN; @@ -2339,23 +2501,16 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et *objp = make_fun(funp); /* Creator pid */ - switch(*ep) { - case PID_EXT: - ep = dec_pid(edep, hpp, ++ep, off_heap, &funp->creator); - if (ep == NULL) { - funp->creator = NIL; /* Don't leave a hole in the heap */ - goto error; - } - break; - default: + if (*ep != PID_EXT + || (ep = dec_pid(edep, hpp, ++ep, off_heap, + &funp->creator))==NULL) { goto error; } /* Module */ - if ((ep = dec_atom(edep, ep, &temp)) == NULL) { + if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - module = temp; /* Index */ if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { @@ -2382,7 +2537,6 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et funp->next = off_heap->funs; off_heap->funs = funp; #endif - old_uniq = unsigned_val(temp); funp->fe = erts_put_fun_entry(module, old_uniq, old_index); @@ -2401,12 +2555,15 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et } default: error: - /* - * Be careful to return the updated heap pointer, to avoid - * that the caller wipes out binaries or other off-heap objects - * that may have been linked into the process. + /* UNDO: + * Must unlink all off-heap objects that may have been + * linked into the process. */ - *hpp = hp; + if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ + hp = *hpp; /* the largest must be the freshest */ + } + undo_offheap_in_area(off_heap, hp_saved, hp); + *hpp = hp_saved; return NULL; } } diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index f308680f89..eada6d4f27 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -98,9 +98,19 @@ typedef struct { Eterm atom[ERTS_ATOM_CACHE_SIZE]; } ErtsAtomTranslationTable; -#define ERTS_DIST_EXT_DFLAG_HDR (((Uint32) 1) << 31) -#define ERTS_DIST_EXT_ATOM_TRANS_TAB (((Uint32) 1) << 30) -#define ERTS_DIST_EXT_CON_ID_MASK ((Uint32) 0x3fffffff) +/* + * These flags are tagged onto the high bits of a connection ID and stored in + * the ErtsDistExternal structure's flags field. They are used to indicate + * various bits of state necessary to decode binaries in a variety of + * scenarios. The mask ERTS_DIST_EXT_CON_ID_MASK is used later to separate the + * connection ID from the flags. Be careful to ensure that the mask does not + * overlap any of the bits used for flags, or ERTS will leak flags bits into + * connection IDs and leak connection ID bits into the flags. + */ +#define ERTS_DIST_EXT_DFLAG_HDR ((Uint32) 0x80000000) +#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x40000000) +#define ERTS_DIST_EXT_BTT_SAFE ((Uint32) 0x20000000) +#define ERTS_DIST_EXT_CON_ID_MASK ((Uint32) 0x1fffffff) #define ERTS_DIST_EXT_CON_ID(DIST_EXTP) \ ((DIST_EXTP)->flags & ERTS_DIST_EXT_CON_ID_MASK) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 1b64e23174..df0ab40074 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -86,7 +86,8 @@ struct enif_environment_t /* ErlNifEnv */ }; extern void erts_pre_nif(struct enif_environment_t*, Process*, void* nif_data); extern void erts_post_nif(struct enif_environment_t* env); -extern Eterm erts_nif_taints(Process* p); +extern Eterm erts_nif_taints(Process* p); +extern void erts_print_nif_taints(int to, void* to_arg); /* * Port Specific Data. @@ -1657,7 +1658,6 @@ void erts_bif_trace_init(void); /* ** Call_trace uses this API for the parameter matching functions */ - struct erl_heap_fragment* saved_program_buf; #define MatchSetRef(MPSP) \ do { \ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 71cb6a36cc..4b949523fa 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -239,6 +239,11 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); ** Sint16: A signed integer of 16 bits exactly. */ +#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \ + && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG))) +#error Cannot handle this combination of int/long/void*/size_t sizes +#endif + #if SIZEOF_VOID_P == 8 #undef ARCH_32 #define ARCH_64 diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index be442fa480..c162395159 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -95,6 +95,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq) #endif + Eterm* erts_heap_alloc(Process* p, Uint need) { @@ -105,13 +106,29 @@ erts_heap_alloc(Process* p, Uint need) Uint i; #endif +#ifdef FORCE_HEAP_FRAGS + if (p->space_verified && p->space_verified_from!=NULL + && HEAP_TOP(p) >= p->space_verified_from + && HEAP_TOP(p) + need <= p->space_verified_from + p->space_verified + && HEAP_LIMIT(p) - HEAP_TOP(p) >= need) { + + Uint consumed = need + (HEAP_TOP(p) - p->space_verified_from); + ASSERT(consumed <= p->space_verified); + p->space_verified -= consumed; + p->space_verified_from += consumed; + HEAP_TOP(p) = p->space_verified_from; + return HEAP_TOP(p) - need; + } + p->space_verified = 0; + p->space_verified_from = NULL; +#endif /* FORCE_HEAP_FRAGS */ + n = need; #ifdef DEBUG n++; #endif bp = (ErlHeapFragment*) - ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG, - sizeof(ErlHeapFragment) + ((n-1)*sizeof(Eterm))); + ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG, ERTS_HEAP_FRAG_SIZE(n)); #ifdef DEBUG n--; @@ -140,6 +157,7 @@ erts_heap_alloc(Process* p, Uint need) bp->next = MBUF(p); MBUF(p) = bp; bp->size = n; + bp->used_size = n; MBUF_SIZE(p) += n; bp->off_heap.mso = NULL; #ifndef HYBRID /* FIND ME! */ @@ -151,34 +169,6 @@ erts_heap_alloc(Process* p, Uint need) return bp->mem; } -void erts_arith_shrink(Process* p, Eterm* hp) -{ -#if defined(CHECK_FOR_HOLES) - ErlHeapFragment* hf; - - /* - * We must find the heap fragment that hp points into. - * If we are unlucky, we might have to search through - * a large part of the list. We'll hope that will not - * happen too often. - */ - for (hf = MBUF(p); hf != 0; hf = hf->next) { - if (hp - hf->mem < (unsigned long)hf->size) { - /* - * We are not allowed to changed hf->size (because the - * size must be correct when deallocating). Therefore, - * clear out the uninitialized part of the heap fragment. - */ - Eterm* to = hf->mem + hf->size; - while (hp < to) { - *hp++ = NIL; - } - break; - } - } -#endif -} - #ifdef CHECK_FOR_HOLES Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz) |