diff options
57 files changed, 806 insertions, 663 deletions
diff --git a/erts/configure.in b/erts/configure.in index 334ee4bd1d..7db501e5b9 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2775,7 +2775,7 @@ if test X${enable_fp_exceptions} != Xyes ; then FPE=unreliable else - AC_MSG_CHECKING([for unreliable floating point execptions]) + AC_MSG_CHECKING([for unreliable floating point exceptions]) AC_TRY_RUN([ diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index f62d3fb170..eb1d24cf12 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -393,9 +393,11 @@ <tag><c><![CDATA[-pa Dir1 Dir2 ...]]></c></tag> <item> <p>Adds the specified directories to the beginning of the code - path, similar to <c><![CDATA[code:add_pathsa/1]]></c>; see - <seealso marker="kernel:code"><c>code(3)</c></seealso>. - As an alternative to <c>-pa</c>, if several directories are + path, similar to <seealso marker="kernel:code#add_pathsa/1"> + <c><![CDATA[code:add_pathsa/1]]></c></seealso>. Note that the + order of the given directories will be reversed in the + resulting path.</p> + <p>As an alternative to <c>-pa</c>, if several directories are to be prepended to the code path and the directories have a common parent directory, that parent directory can be specified in environment variable <c>ERL_LIBS</c>; see diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index b5dc9037c4..906c1be17b 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -80,7 +80,7 @@ <code type="none"> /* niftest.c */ -#include "erl_nif.h" +#include <erl_nif.h> static ERL_NIF_TERM hello(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { @@ -296,8 +296,8 @@ return term;</code> synchronization. This includes terms in process-independent environments that are shared between threads. Resource objects also require synchronization if you treat them as mutable.</p> - <p>The library initialization callbacks <c>load</c>, <c>reload</c>, and - <c>upgrade</c> are all thread-safe even for shared state data.</p> + <p>The library initialization callbacks <c>load</c> and + <c>upgrade</c> are thread-safe even for shared state data.</p> </item> <tag><marker id="version_management"/>Version Management</tag> <item> @@ -498,7 +498,7 @@ return term;</code> <title>Initialization</title> <taglist> <tag><marker id="ERL_NIF_INIT"/><c>ERL_NIF_INIT(MODULE, - ErlNifFunc funcs[], load, reload, upgrade, unload)</c></tag> + ErlNifFunc funcs[], load, NULL, upgrade, unload)</c></tag> <item> <p>This is the magic macro to initialize a NIF library. It is to be evaluated in global file scope.</p> @@ -507,11 +507,14 @@ return term;</code> the macro.</p> <p><c>funcs</c> is a static array of function descriptors for all the implemented NIFs in this library.</p> - <p><c>load</c>, <c>reload</c>, <c>upgrade</c> and <c>unload</c> - are pointers to functions. One of <c>load</c>, <c>reload</c>, or + <p><c>load</c>, <c>upgrade</c> and <c>unload</c> + are pointers to functions. One of <c>load</c> or <c>upgrade</c> is called to initialize the library. <c>unload</c> is called to release the library. All are described individually below.</p> + <p>The fourth argument <c>NULL</c> is ignored. It + was earlier used for the deprectated <c>reload</c> callback + which is no longer supported since OTP 20.</p> <p>If compiling a NIF for static inclusion through <c>--enable-static-nifs</c>, you must define <c>STATIC_ERLANG_NIF</c> before the <c>ERL_NIF_INIT</c> declaration.</p> @@ -539,7 +542,7 @@ return term;</code> and there is old code of this module with a loaded NIF library.</p> <p>Works as <c>load</c>, except that <c>*old_priv_data</c> already contains the value set by the last call to <c>load</c> or - <c>reload</c> for the old module code. <c>*priv_data</c> is + <c>upgrade</c> for the old module code. <c>*priv_data</c> is initialized to <c>NULL</c> when <c>upgrade</c> is called. It is allowed to write to both <c>*priv_data</c> and <c>*old_priv_data.</c></p> @@ -551,27 +554,7 @@ return term;</code> <item> <p><c>unload</c> is called when the module code that the NIF library belongs to is purged as old. New code of the same - module may or may not exist. Notice that <c>unload</c> is not - called for a replaced library as a consequence of <c>reload</c>.</p> - </item> - <tag><marker id="reload"/><c>int (*reload)(ErlNifEnv* env, void** - priv_data, ERL_NIF_TERM load_info)</c></tag> - <item> - <note> - <p><em>The reload mechanism is deprecated.</em> It was only intended - as a development feature. Do not use it as an upgrade method for - live production systems. It can be removed in future releases. - Ensure to pass <c>reload</c> as <c>NULL</c> to - <seealso marker="#ERL_NIF_INIT"><c>ERL_NIF_INIT</c></seealso> - to disable it when not used.</p> - </note> - <p><c>reload</c> is called when the NIF library is loaded and a - previously loaded library already exists for this module code.</p> - <p>Works as <c>load</c>, except that - <c>*priv_data</c> already contains the value set by the - previous call to <c>load</c> or <c>reload</c>.</p> - <p>The library fails to load if <c>reload</c> returns - anything other than <c>0</c> or if <c>reload</c> is <c>NULL</c>.</p> + module may or may not exist.</p> </item> </taglist> </section> @@ -2249,9 +2232,8 @@ enif_map_iterator_destroy(env, &iter);</code> returns <c>NULL</c> and sets <c>*tried</c> to <c>flags</c>. It is allowed to set <c>tried</c> to <c>NULL</c>.</p> <p>Notice that <c>enif_open_resource_type</c> is only allowed to be - called in the three callbacks - <seealso marker="#load"><c>load</c></seealso>, - <seealso marker="#reload"><c>reload</c></seealso>, and + called in the two callbacks + <seealso marker="#load"><c>load</c></seealso> and <seealso marker="#upgrade"><c>upgrade</c></seealso>.</p> </desc> </func> @@ -2305,10 +2287,8 @@ enif_map_iterator_destroy(env, &iter);</code> <fsummary>Get the private data of a NIF library.</fsummary> <desc> <p>Returns the pointer to the private data that was set by - <seealso marker="#load"><c>load</c></seealso>, - <seealso marker="#reload"><c>reload</c></seealso>, or + <seealso marker="#load"><c>load</c></seealso> or <seealso marker="#upgrade"><c>upgrade</c></seealso>.</p> - <p>Was previously named <c>enif_get_data</c>.</p> </desc> </func> diff --git a/erts/doc/src/erl_tracer.xml b/erts/doc/src/erl_tracer.xml index 83eef374ca..43613c31b1 100644 --- a/erts/doc/src/erl_tracer.xml +++ b/erts/doc/src/erl_tracer.xml @@ -684,7 +684,7 @@ trace(_, _, _, _, _) -> <p><c>erl_msg_tracer.c</c>:</p> <pre> -#include "erl_nif.h" +#include <erl_nif.h> /* NIF interface declarations */ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info); diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index f42c39db5f..f318f28b6d 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -2562,13 +2562,6 @@ os_prompt%</pre> <name name="load_nif" arity="2"/> <fsummary>Load NIF library.</fsummary> <desc> - <note> - <p>Before Erlang/OTP R14B, NIFs were an - experimental feature. Versions before Erlang/OTP R14B can - have different and possibly incompatible NIF semantics and - interfaces. For example, in Erlang/OTP R13B03 the return value on - failure was <c>{error,Reason,Text}</c>.</p> - </note> <p>Loads and links a dynamic library containing native implemented functions (NIFs) for a module. <c><anno>Path</anno></c> is a file path to the shareable object/dynamic library file minus @@ -2598,9 +2591,13 @@ os_prompt%</pre> <item>The library did not fulfill the requirements as a NIF library of the calling module. </item> - <tag><c>load | reload | upgrade</c></tag> + <tag><c>load | upgrade</c></tag> <item>The corresponding library callback was unsuccessful. </item> + <tag><c>reload</c></tag> + <item>A NIF library is already loaded for this module instance. + The previously deprecated <c>reload</c> feature was removed in OTP 20. + </item> <tag><c>old_code</c></tag> <item>The call to <c>load_nif/2</c> was made from the old code of a module that has been upgraded; this is not diff --git a/erts/doc/src/run_erl.xml b/erts/doc/src/run_erl.xml index ad7b2c5b85..a9b6a7e2c6 100644 --- a/erts/doc/src/run_erl.xml +++ b/erts/doc/src/run_erl.xml @@ -49,7 +49,7 @@ <funcs> <func> <name>run_erl [-daemon] pipe_dir/ log_dir "exec command - [command_arguments]"</name> + arg1 arg2 ..."</name> <fsummary>Start the Erlang emulator without attached terminal.</fsummary> <desc> <p>Arguments:</p> @@ -92,11 +92,10 @@ </item> </list> </item> - <tag><c>"exec command [command_arguments]"</c></tag> + <tag><c>"exec command arg1 arg2 ..."</c></tag> <item> - <p>In the third argument, <c><![CDATA[command]]></c> is the - executable to execute where everything written to <c>stdin</c> - and <c>stdout</c> is logged to <c><![CDATA[log_dir]]></c>.</p> + <p>A space-separated string specifying the program to be executed. + The second field is typically a command name such as <c>erl</c>.</p> </item> </taglist> </desc> @@ -105,7 +104,7 @@ <section> <title>Notes concerning the Log Files</title> - <p>While running, <c>run_erl</c> (as stated earlier) sends all output, + <p>While running, <c>run_erl</c> sends all output, uninterpreted, to a log file. The file is named <c><![CDATA[erlang.log.N]]></c>, where <c>N</c> is an integer. When the log is "full" (default log size is 100 KB), <c>run_erl</c> starts to log diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index cb7278696f..75389107bb 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3542,8 +3542,14 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("DbTable_words", BIF_ARG_1)) { /* Used by ets_SUITE (stdlib) */ size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint); - BIF_RET(make_small((Uint) words)); + Eterm* hp = HAlloc(BIF_P ,3); + BIF_RET(TUPLE2(hp, make_small((Uint) words), + erts_ets_hash_sizeof_ext_segtab())); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_get_meta_state(BIF_P)); + } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { /* Used by driver_SUITE (emulator) */ Uint sz, *szp; @@ -4277,6 +4283,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } BIF_RET(am_ok); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2)); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 128a7b3865..485f68ca98 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3978,6 +3978,36 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } +/* + * For testing only + * Retreive meta table size state + */ +Eterm erts_ets_get_meta_state(Process* p) +{ + Eterm* hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash), + erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash)); +} +/* + * For testing only + * Restore a previously retrieved meta table size state. + * We do this to suppress failed memory checks + * caused by the hysteresis of meta tables grow/shrink limits. + */ +Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state) +{ + Eterm* tv; + Eterm* hp; + if (!is_tuple_arity(meta_state, 2)) + return am_badarg; + + tv = tuple_val(meta_state); + hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]), + erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2])); +} #ifdef HARDDEBUG /* Here comes some debug functions */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 1d26c49652..f7eb3dc45c 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -90,6 +90,8 @@ extern Export ets_select_continue_exp; extern erts_smp_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); +Eterm erts_ets_get_meta_state(Process* p); +Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state); Uint erts_db_get_max_tabs(void); @@ -267,7 +269,5 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#undef ERTS_DB_ALC_MEM_UPDATE_ - #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 5e6fe4f460..939ae57192 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -84,25 +84,28 @@ #include "erl_db_hash.h" -#ifdef MYDEBUG /* Will fail test case ets_SUITE:memory */ -# define IF_DEBUG(x) x -# define MY_ASSERT(x) ASSERT(x) -#else -# define IF_DEBUG(x) -# define MY_ASSERT(x) -#endif - /* * The following symbols can be manipulated to "tune" the linear hash array */ -#define CHAIN_LEN 6 /* Medium bucket chain len */ +#define GROW_LIMIT(NACTIVE) ((NACTIVE)*1) +#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) -/* Number of slots per segment */ -#define SEGSZ_EXP 8 -#define SEGSZ (1 << SEGSZ_EXP) -#define SEGSZ_MASK (SEGSZ-1) +/* +** We want the first mandatory segment to be small (to reduce minimal footprint) +** and larger extra segments (to reduce number of alloc/free calls). +*/ -#define NSEG_1 2 /* Size of first segment table (must be at least 2) */ +/* Number of slots in first segment */ +#define FIRST_SEGSZ_EXP 8 +#define FIRST_SEGSZ (1 << FIRST_SEGSZ_EXP) +#define FIRST_SEGSZ_MASK (FIRST_SEGSZ - 1) + +/* Number of slots per extra segment */ +#define EXT_SEGSZ_EXP 11 +#define EXT_SEGSZ (1 << EXT_SEGSZ_EXP) +#define EXT_SEGSZ_MASK (EXT_SEGSZ-1) + +#define NSEG_1 (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*)) #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ @@ -123,7 +126,9 @@ #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) -#define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK] +#define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP) + +#define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK] /* * When deleting a table, the number of records to delete. @@ -189,6 +194,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) # define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck) +# define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval)) /* Fine grained read lock */ static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval) @@ -318,27 +324,23 @@ struct mp_info { /* A table segment */ struct segment { - HashDbTerm* buckets[SEGSZ]; -#ifdef MYDEBUG - int is_ext_segment; -#endif + HashDbTerm* buckets[1]; }; +#define SIZEOF_SEGMENT(N) \ + (offsetof(struct segment,buckets) + sizeof(HashDbTerm*)*(N)) -/* A segment that also contains a segment table */ -struct ext_segment { - struct segment s; /* The segment itself. Must be first */ - +/* An extended segment table */ +struct ext_segtab { +#ifdef ERTS_SMP + ErtsThrPrgrLaterOp lop; +#endif struct segment** prev_segtab; /* Used when table is shrinking */ - int nsegs; /* Size of segtab */ + int prev_nsegs; /* Size of prev_segtab */ + int nsegs; /* Size of this segtab */ struct segment* segtab[1]; /* The segment table */ }; -#define SIZEOF_EXTSEG(NSEGS) \ - (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS)) - -#if defined(DEBUG) || defined(VALGRIND) -# define EXTSEG(SEGTAB_PTR) \ - ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab))) -#endif +#define SIZEOF_EXT_SEGTAB(NSEGS) \ + (offsetof(struct ext_segtab,segtab) + sizeof(struct segment*)*(NSEGS)) static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, @@ -348,49 +350,22 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); else erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); -#ifdef VALGRIND - tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab); -#endif } -/* How the table segments relate to each other: - - ext_segment: ext_segment: "plain" segment - #=================# #================# #=============# - | bucket[0] |<--+ +------->| bucket[256] | +->| bucket[512] | - | bucket[1] | | | | [257] | | | [513] | - : : | | : : | : : - | bucket[255] | | | | [511] | | | [767] | - |-----------------| | | |----------------| | #=============# - | prev_segtab=NULL| | | +--<---prev_segtab | | - | nsegs = 2 | | | | | nsegs = 256 | | -+->| segtab[0] -->-------+---|---|--<---segtab[0] |<-+ | -| | segtab[1] -->-----------+---|--<---segtab[1] | | | -| #=================# | | segtab[2] -->-----|--+ ext_segment: -| | : : | #================# -+----------------<---------------+ | segtab[255] ->----|----->| bucket[255*256]| - #================# | | | - | : : - | |----------------| - +----<---prev_segtab | - : : -*/ - /* ** Forward decl's (static functions) */ -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab); -static int alloc_seg(DbTableHash *tb); +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); +static void alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); -static void shrink(DbTableHash* tb, int nactive); -static void grow(DbTableHash* tb, int nactive); +static void shrink(DbTableHash* tb, int nitems); +static void grow(DbTableHash* tb, int nitems); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, @@ -463,9 +438,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + int nitems = NITEMS(tb); + if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { - shrink(tb, nactive); + shrink(tb, nitems); } } @@ -662,16 +638,20 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); - erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); + erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); + erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); - SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab); + SET_SEGTAB(tb, tb->first_segtab); tb->nsegs = NSEG_1; - tb->nslots = SEGSZ; + tb->nslots = FIRST_SEGSZ; + tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(FIRST_SEGSZ)); + sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ)); - erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP + erts_smp_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; @@ -686,7 +666,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, "db_hash_slot", make_small(i)); } - /* This important property is needed to guarantee that the buckets + /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: */ ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); @@ -862,8 +842,8 @@ Lnew: WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) { - grow(tb, nactive); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); } } CHECK_TABLES(); @@ -1514,6 +1494,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; } current = current->next; } @@ -1531,7 +1512,6 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, } } else { /* Key is variable */ - --num_left; if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) { slot_ix = -1; @@ -2250,12 +2230,12 @@ static int db_free_table_continue_hash(DbTable *tbl) done /= 2; while(tb->nslots != 0) { - free_seg(tb, 1); + done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) { + if (done >= DELETE_RECORD_LIMIT) { return 0; /* Not done */ } } @@ -2414,90 +2394,81 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, return DB_ERROR_NONE; } -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab) +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) { - int nsegs; - struct ext_segment* eseg; + struct segment** old_segtab = SEGTAB(tb); + int nsegs = 0; + struct ext_segtab* est; + ASSERT(seg_ix >= NSEG_1); switch (seg_ix) { - case 0: nsegs = NSEG_1; break; - case 1: nsegs = NSEG_2; break; - default: nsegs = seg_ix + NSEG_INC; break; - } - eseg = (struct ext_segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - SIZEOF_EXTSEG(nsegs)); - ASSERT(eseg != NULL); - sys_memset(&eseg->s, 0, sizeof(struct segment)); - IF_DEBUG(eseg->s.is_ext_segment = 1); - eseg->prev_segtab = old_segtab; - eseg->nsegs = nsegs; - if (old_segtab) { - ASSERT(nsegs > tb->nsegs); - sys_memcpy(eseg->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); - } + case NSEG_1: nsegs = NSEG_2; break; + default: nsegs = seg_ix + NSEG_INC; break; + } + ASSERT(nsegs > tb->nsegs); + est = (struct ext_segtab*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_EXT_SEGTAB(nsegs)); + est->nsegs = nsegs; + est->prev_segtab = old_segtab; + est->prev_nsegs = tb->nsegs; + sys_memcpy(est->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); #ifdef DEBUG - sys_memset(&eseg->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); + sys_memset(&est->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); #endif - eseg->segtab[seg_ix] = &eseg->s; - return eseg; + return est; } /* Extend table with one new segment */ -static int alloc_seg(DbTableHash *tb) +static void alloc_seg(DbTableHash *tb) { - int seg_ix = tb->nslots >> SEGSZ_EXP; - - if (seg_ix+1 == tb->nsegs) { /* New segtab needed (extended segment) */ - struct segment** segtab = SEGTAB(tb); - struct ext_segment* seg = alloc_ext_seg(tb, seg_ix, segtab); - if (seg == NULL) return 0; - segtab[seg_ix] = &seg->s; - /* We don't use the new segtab until next call (see "shrink race") */ - } - else { /* Just a new plain segment */ - struct segment** segtab; - if (seg_ix == tb->nsegs) { /* Time to start use segtab from last call */ - struct ext_segment* eseg; - eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; - MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - SET_SEGTAB(tb, eseg->segtab); - tb->nsegs = eseg->nsegs; - } - ASSERT(seg_ix < tb->nsegs); - segtab = SEGTAB(tb); - ASSERT(segtab[seg_ix] == NULL); - segtab[seg_ix] = (struct segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - sizeof(struct segment)); - if (segtab[seg_ix] == NULL) return 0; - sys_memset(segtab[seg_ix], 0, sizeof(struct segment)); - } - tb->nslots += SEGSZ; - return 1; + int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots); + struct segment** segtab; + + ASSERT(seg_ix > 0); + if (seg_ix == tb->nsegs) { /* New segtab needed */ + struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix); + SET_SEGTAB(tb, est->segtab); + tb->nsegs = est->nsegs; + } + ASSERT(seg_ix < tb->nsegs); + segtab = SEGTAB(tb); + segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(EXT_SEGSZ)); + sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); + tb->nslots += EXT_SEGSZ; +} + +#ifdef ERTS_SMP +static void dealloc_ext_segtab(void* lop_data) +{ + struct ext_segtab* est = (struct ext_segtab*) lop_data; + + erts_free(ERTS_ALC_T_DB_SEG, est); } +#endif /* Shrink table by freeing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty */ static int free_seg(DbTableHash *tb, int free_records) { - const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1; struct segment** const segtab = SEGTAB(tb); - struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; - int bytes; + struct segment* const segp = segtab[seg_ix]; + Uint seg_sz; int nrecords = 0; - ASSERT(top != NULL); + ASSERT(segp != NULL); #ifndef DEBUG if (free_records) #endif { - int i; - for (i=0; i<SEGSZ; ++i) { - HashDbTerm* p = top->s.buckets[i]; + int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + while (i--) { + HashDbTerm* p = segp->buckets[i]; while(p != 0) { HashDbTerm* nxt = p->next; ASSERT(free_records); /* segment not empty as assumed? */ @@ -2507,55 +2478,46 @@ static int free_seg(DbTableHash *tb, int free_records) } } } - - /* The "shrink race": - * We must avoid deallocating an extended segment while its segtab may - * still be used by other threads. - * The trick is to stop use a segtab one call earlier. That is, stop use - * a segtab when the segment above it is deallocated. When the segtab is - * later deallocated, it has not been used for a very long time. - * It is even theoretically safe as we have by then rehashed the entire - * segment, seizing *all* locks, so there cannot exist any retarded threads - * still hanging in BUCKET macro with an old segtab pointer. - * For this to work, we must of course allocate a new segtab one call - * earlier in alloc_seg() as well. And this is also the reason why - * the minimum size of the first segtab is 2 and not 1 (NSEG_1). - */ - if (seg_ix == tb->nsegs-1 || seg_ix==0) { /* Dealloc extended segment */ - MY_ASSERT(top->s.is_ext_segment); - ASSERT(segtab != top->segtab || seg_ix==0); - bytes = SIZEOF_EXTSEG(top->nsegs); - } - else { /* Dealloc plain segment */ - struct ext_segment* newtop = (struct ext_segment*) segtab[seg_ix-1]; - MY_ASSERT(!top->s.is_ext_segment); - - if (segtab == newtop->segtab) { /* New top segment is extended */ - MY_ASSERT(newtop->s.is_ext_segment); - if (newtop->prev_segtab != NULL) { - /* Time to use a smaller segtab */ - SET_SEGTAB(tb, newtop->prev_segtab); - tb->nsegs = seg_ix; - ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); - } - else { - ASSERT(NSEG_1 > 2 && seg_ix==1); - } - } - bytes = sizeof(struct segment); + if (seg_ix >= NSEG_1) { + struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab); + + if (seg_ix == est->prev_nsegs) { /* Dealloc extended segtab */ + ASSERT(est->prev_segtab != NULL); + SET_SEGTAB(tb, est->prev_segtab); + tb->nsegs = est->prev_nsegs; + +#ifdef ERTS_SMP + if (!tb->common.is_thread_safe) { + /* + * Table is doing a graceful shrink operation and we must avoid + * deallocating this segtab while it may still be read by other + * threads. Schedule deallocation with thread progress to make + * sure no lingering threads are still hanging in BUCKET macro + * with an old segtab pointer. + */ + Uint sz = SIZEOF_EXT_SEGTAB(est->nsegs); + ASSERT(sz == ERTS_ALC_DBG_BLK_SZ(est)); + ERTS_DB_ALC_MEM_UPDATE_(tb, sz, 0); + erts_schedule_thr_prgr_later_cleanup_op(dealloc_ext_segtab, + est, + &est->lop, + sz); + } + else +#endif + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } } + seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, - (void*)top, bytes); #ifdef DEBUG - if (seg_ix > 0) { - segtab[seg_ix] = NULL; - } else { - SET_SEGTAB(tb, NULL); - } + if (seg_ix < tb->nsegs) + SEGTAB(tb)[seg_ix] = NULL; #endif - tb->nslots -= SEGSZ; + tb->nslots -= seg_sz; ASSERT(tb->nslots >= 0); return nrecords; } @@ -2604,104 +2566,111 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1); - else { - if (erts_smp_atomic_read_nob(&tb->is_resizing)) - return 0; - erts_smp_atomic_set_nob(&tb->is_resizing, 1); - return 1; - } + return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + else + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); +#endif + return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->is_resizing, 0); - else - erts_smp_atomic_set_nob(&tb->is_resizing, 0); + erts_atomic_set_relb(&tb->is_resizing, 0); +#endif } -/* Grow table with one new bucket. +/* Grow table with one or more new buckets. ** Allocate new segment if needed. */ -static void grow(DbTableHash* tb, int nactive) +static void grow(DbTableHash* tb, int nitems) { HashDbTerm** pnext; HashDbTerm** to_pnext; HashDbTerm* p; erts_smp_rwmtx_t* lck; - int from_ix; + int nactive; + int from_ix, to_ix; int szm; + int loop_limit = 5; - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) != nactive) { - goto abort; /* already done (race) */ - } - - /* Ensure that the slot nactive exists */ - if (nactive == tb->nslots) { - /* Time to get a new segment */ - ASSERT((nactive & SEGSZ_MASK) == 0); - if (!alloc_seg(tb)) goto abort; - } - ASSERT(nactive < tb->nslots); + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (nitems <= GROW_LIMIT(nactive)) { + goto abort; /* already done (race) */ + } - szm = erts_smp_atomic_read_nob(&tb->szm); - if (nactive <= szm) { - from_ix = nactive & (szm >> 1); - } else { - ASSERT(nactive == szm+1); - from_ix = 0; - szm = (szm<<1) | 1; - } + /* Ensure that the slot nactive exists */ + if (nactive == tb->nslots) { + /* Time to get a new segment */ + ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0); + alloc_seg(tb); + } + ASSERT(nactive < tb->nslots); - lck = WLOCK_HASH(tb, from_ix); - /* Now a final double check (with the from_ix lock held) - * that we did not get raced by a table fixer. - */ - if (IS_FIXED(tb)) { - WUNLOCK_HASH(lck); - goto abort; - } - erts_smp_atomic_inc_nob(&tb->nactive); - if (from_ix == 0) { - if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->szm, szm); - else - erts_smp_atomic_set_nob(&tb->szm, szm); - } - done_resizing(tb); + szm = erts_smp_atomic_read_nob(&tb->szm); + if (nactive <= szm) { + from_ix = nactive & (szm >> 1); + } else { + ASSERT(nactive == szm+1); + from_ix = 0; + szm = (szm<<1) | 1; + } + to_ix = nactive; + + lck = WLOCK_HASH(tb, from_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix)); + /* Now a final double check (with the from_ix lock held) + * that we did not get raced by a table fixer. + */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + erts_smp_atomic_set_nob(&tb->nactive, ++nactive); + if (from_ix == 0) { + if (DB_USING_FINE_LOCKING(tb)) + erts_smp_atomic_set_relb(&tb->szm, szm); + else + erts_smp_atomic_set_nob(&tb->szm, szm); + } + done_resizing(tb); + + /* Finally, let's split the bucket. We try to do it in a smart way + to keep link order and avoid unnecessary updates of next-pointers */ + pnext = &BUCKET(tb, from_ix); + p = *pnext; + to_pnext = &BUCKET(tb, to_ix); + while (p != NULL) { + if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ + *pnext = p->next; + free_term(tb, p); + p = *pnext; + } + else { + int ix = p->hvalue & szm; + if (ix != from_ix) { + ASSERT(ix == (from_ix ^ ((szm+1)>>1))); + *to_pnext = p; + /* Swap "from" and "to": */ + from_ix = ix; + to_pnext = pnext; + } + pnext = &p->next; + p = *pnext; + } + } + *to_pnext = NULL; + WUNLOCK_HASH(lck); - /* Finally, let's split the bucket. We try to do it in a smart way - to keep link order and avoid unnecessary updates of next-pointers */ - pnext = &BUCKET(tb, from_ix); - p = *pnext; - to_pnext = &BUCKET(tb, nactive); - while (p != NULL) { - if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ - *pnext = p->next; - free_term(tb, p); - p = *pnext; - } - else { - int ix = p->hvalue & szm; - if (ix != from_ix) { - ASSERT(ix == (from_ix ^ ((szm+1)>>1))); - *to_pnext = p; - /* Swap "from" and "to": */ - from_ix = ix; - to_pnext = pnext; - } - pnext = &p->next; - p = *pnext; - } - } - *to_pnext = NULL; + }while (--loop_limit && nitems > GROW_LIMIT(nactive)); - WUNLOCK_HASH(lck); return; abort: @@ -2712,60 +2681,78 @@ abort: /* Shrink table by joining top bucket. ** Remove top segment if it gets empty. */ -static void shrink(DbTableHash* tb, int nactive) -{ - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) == nactive) { - erts_smp_rwmtx_t* lck; - int src_ix = nactive - 1; - int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; - int dst_ix = src_ix & low_szm; - - ASSERT(dst_ix < src_ix); - ASSERT(nactive > SEGSZ); - lck = WLOCK_HASH(tb, dst_ix); - /* Double check for racing table fixers */ - if (!IS_FIXED(tb)) { - HashDbTerm** src_bp = &BUCKET(tb, src_ix); - HashDbTerm** dst_bp = &BUCKET(tb, dst_ix); - HashDbTerm** bp = src_bp; - - /* Q: Why join lists by appending "dst" at the end of "src"? - A: Must step through "src" anyway to purge pseudo deleted. */ - while(*bp != NULL) { - if ((*bp)->hvalue == INVALID_HASH) { - HashDbTerm* deleted = *bp; - *bp = deleted->next; - free_term(tb, deleted); - } else { - bp = &(*bp)->next; - } - } - *bp = *dst_bp; - *dst_bp = *src_bp; - *src_bp = NULL; - - erts_smp_atomic_set_nob(&tb->nactive, src_ix); - if (dst_ix == 0) { - erts_smp_atomic_set_relb(&tb->szm, low_szm); - } - WUNLOCK_HASH(lck); - - if (tb->nslots - src_ix >= SEGSZ) { - free_seg(tb, 0); - } - } - else { - WUNLOCK_HASH(lck); - } +static void shrink(DbTableHash* tb, int nitems) +{ + HashDbTerm** src_bp; + HashDbTerm** dst_bp; + HashDbTerm** bp; + erts_smp_rwmtx_t* lck; + int src_ix, dst_ix, low_szm; + int nactive; + int loop_limit = 5; - } - /*else already done */ + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) { + goto abort; /* already done (race) */ + } + src_ix = nactive - 1; + low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; + dst_ix = src_ix & low_szm; + + ASSERT(dst_ix < src_ix); + ASSERT(nactive > FIRST_SEGSZ); + lck = WLOCK_HASH(tb, dst_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix)); + /* Double check for racing table fixers */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + + src_bp = &BUCKET(tb, src_ix); + dst_bp = &BUCKET(tb, dst_ix); + bp = src_bp; + + /* + * We join lists by appending "dst" at the end of "src" + * as we must step through "src" anyway to purge pseudo deleted. + */ + while(*bp != NULL) { + if ((*bp)->hvalue == INVALID_HASH) { + HashDbTerm* deleted = *bp; + *bp = deleted->next; + free_term(tb, deleted); + } else { + bp = &(*bp)->next; + } + } + *bp = *dst_bp; + *dst_bp = *src_bp; + *src_bp = NULL; + + nactive = src_ix; + erts_smp_atomic_set_nob(&tb->nactive, nactive); + if (dst_ix == 0) { + erts_smp_atomic_set_relb(&tb->szm, low_szm); + } + WUNLOCK_HASH(lck); + + if (tb->nslots - src_ix >= EXT_SEGSZ) { + free_seg(tb, 0); + } + done_resizing(tb); + + } while (--loop_limit + && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)); + return; + +abort: done_resizing(tb); } - /* Search a list of tuples for a matching key */ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, @@ -2931,8 +2918,8 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) WUNLOCK_HASH(lck); nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { - grow(tb, nactive); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); } } else { WUNLOCK_HASH(lck); @@ -3016,6 +3003,51 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); stats->kept_items = kept_items; } + +/* For testing only */ +Eterm erts_ets_hash_sizeof_ext_segtab(void) +{ + return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); +} +/* For testing only */ +Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) +{ + Eterm seg_cnt; + while (!begin_resizing(tb)) + /*spinn*/; + + seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots)); + done_resizing(tb); + return seg_cnt; +} +/* For testing only */ +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) +{ + int seg_cnt, target; + + if (!is_small(memstate)) + return make_small(__LINE__); + + target = signed_val(memstate); + if (target < 1) + return make_small(__LINE__); + while (1) { + while (!begin_resizing(tb)) + /*spin*/; + seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots); + done_resizing(tb); + + if (target == seg_cnt) + return am_ok; + if (IS_FIXED(tb)) + return make_small(__LINE__); + if (target < seg_cnt) + shrink(tb, 0); + else + grow(tb, INT_MAX); + } +} + #ifdef HARDDEBUG void db_check_table_hash(DbTable *tbl) @@ -3024,7 +3056,7 @@ void db_check_table_hash(DbTable *tbl) HashDbTerm* list; int j; - for (j = 0; j < tb->nactive; j++) { + for (j = 0; j < NACTIVE(tb); j++) { if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e654363cd5..6d25c73549 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -50,23 +50,23 @@ typedef struct db_table_hash_fine_locks { typedef struct db_table_hash { DbTableCommon common; - erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ erts_smp_atomic_t szm; /* current size mask. */ - + erts_smp_atomic_t nactive; /* Number of "active" slots */ + + erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + struct segment* first_segtab[1]; + /* SMP: nslots and nsegs are protected by is_resizing or table write lock */ int nslots; /* Total number of slots */ int nsegs; /* Size of segment table */ /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ - erts_smp_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ #ifdef ERTS_SMP + erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif -#ifdef VALGRIND - struct ext_segment* top_ptr_to_segment_with_active_segtab; -#endif } DbTableHash; @@ -109,5 +109,8 @@ typedef struct { }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); +Eterm erts_ets_hash_sizeof_ext_segtab(void); +Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb); +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate); #endif /* _DB_HASH_H */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index b53d3d4c19..7442e99cb3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3145,7 +3145,6 @@ static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) BIF_RETTYPE load_nif_2(BIF_ALIST_2) { static const char bad_lib[] = "bad_lib"; - static const char reload[] = "reload"; static const char upgrade[] = "upgrade"; char* lib_name = NULL; void* handle = NULL; @@ -3162,7 +3161,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; - int reload_warning = 0; struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; @@ -3222,8 +3220,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi = module_p->on_load; } - if (init_func == NULL && - (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { + if (this_mi->nif != NULL) { + ret = load_nif_error(BIF_P,"reload","NIF library already loaded" + " (reload disallowed since OTP 20)."); + } + else if (init_func == NULL && + (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); @@ -3312,7 +3314,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) goto error; } - /* Call load, reload or upgrade: + /* Call load or upgrade: */ @@ -3324,82 +3326,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(opened_rt_list == NULL); lib->mod = module_p; env.mod_nif = lib; - if (this_mi->nif != NULL) { /*************** Reload ******************/ - /* - * Repeated load_nif calls from same Erlang module instance ("reload") - * is deprecated and was only ment as a development feature not to - * be used in production systems. (See warning below) - */ - int k, old_incr = 0; - ErlNifFunc* old_func; - lib->priv_data = this_mi->nif->priv_data; - - ASSERT(this_mi->nif->entry != NULL); - if (entry->reload == NULL) { - ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); - goto error; - } - /* Check that no NIF is removed */ - old_func = this_mi->nif->entry->funcs; - for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) { - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs; i++) { - if (old_func->arity == f->arity - && sys_strcmp(old_func->name, f->name) == 0) { - break; - } - f = next_func(entry, &incr, f); - } - if (i == entry->num_of_funcs) { - ret = load_nif_error(BIF_P,reload,"Reloaded library missing " - "function %T:%s/%u\r\n", mod_atom, - old_func->name, old_func->arity); - goto error; - } - old_func = next_func(this_mi->nif->entry, &old_incr, old_func); - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful."); - } - else { - commit_opened_resource_types(lib); - this_mi->nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(this_mi->nif); - reload_warning = 1; - } + + lib->priv_data = NULL; + if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ + void* prev_old_data = prev_mi->nif->priv_data; + if (entry->upgrade == NULL) { + ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); + goto error; + } + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + prev_mi->nif->priv_data = prev_old_data; + ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); + } + else + commit_opened_resource_types(lib); } - else { - lib->priv_data = NULL; - if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ - void* prev_old_data = prev_mi->nif->priv_data; - if (entry->upgrade == NULL) { - ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); - goto error; - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); - } - else - commit_opened_resource_types(lib); - } - else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); - } - else - commit_opened_resource_types(lib); - } + else if (entry->load != NULL) { /********* Initial load ***********/ + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); + } + else + commit_opened_resource_types(lib); } if (ret == am_ok) { /* @@ -3458,14 +3411,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); - if (reload_warning) { - erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Repeated calls to erlang:load_nif from module '%T'.\n\n" - "The NIF reload mechanism is deprecated and must not " - "be used in production systems.\n", mod_atom); - erts_send_warning_to_logger(BIF_P->group_leader, dsbufp); - } BIF_RET(ret); } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index dfe82cab44..16e6c33367 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -99,6 +99,12 @@ #define ErtsContainerStruct(ptr, type, member) \ ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) +/* Use this variant when the member is an array */ +#define ErtsContainerStruct_(ptr, type, memberv) \ + ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv))) + +#define ErtsSizeofMember(type, member) sizeof(((type *)0)->member) + #if defined (__WIN32__) # include "erl_win_sys.h" #else diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h index f0ca91bd06..2b742dd7e3 100644 --- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h @@ -20,7 +20,7 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdlib.h> typedef struct { diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index a0019e5d95..daaff955bc 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <assert.h> #ifdef __WIN32__ #include <windows.h> diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c index e011aadce9..46ee8b5540 100644 --- a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c +++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c @@ -24,7 +24,7 @@ * Author: Rickard Green */ -#include "erl_nif.h" +#include <erl_nif.h> #ifdef __WIN32__ # ifndef WIN32_LEAN_AND_MEAN diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 9c1694fa8a..306f2091a1 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -28,7 +28,7 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2, - basic/1, reload/1, upgrade/1, heap_frag/1, + basic/1, reload_error/1, upgrade/1, heap_frag/1, t_on_load/1, types/1, many_args/1, binaries/1, get_string/1, get_atom/1, maps/1, @@ -68,7 +68,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [basic, reload, upgrade, heap_frag, types, many_args, + [basic, reload_error, upgrade, heap_frag, types, many_args, t_on_load, binaries, get_string, get_atom, maps, api_macros, from_array, iolist_as_binary, resource, resource_binary, @@ -112,8 +112,8 @@ basic(Config) when is_list(Config) -> true = lists:member(?MODULE, erlang:system_info(taints)), ok. -%% Test reload callback in nif lib -reload(Config) when is_list(Config) -> +%% Test old reload feature now always fails +reload_error(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -127,20 +127,20 @@ reload(Config) when is_list(Config) -> hold_nif_mod_priv_data(nif_mod:get_priv_data_ptr()), [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(), - ok = nif_mod:load_nif_lib(Config, 2), - 2 = nif_mod:lib_version(), - [{reload,2,1,201},{lib_version,2,2,202}] = nif_mod_call_history(), + {error, {reload, _}} = nif_mod:load_nif_lib(Config, 2), + 1 = nif_mod:lib_version(), + [{lib_version,1,3,103}] = nif_mod_call_history(), - ok = nif_mod:load_nif_lib(Config, 1), + {error, {reload, _}} = nif_mod:load_nif_lib(Config, 1), 1 = nif_mod:lib_version(), - [{reload,1,1,101},{lib_version,1,2,102}] = nif_mod_call_history(), + [{lib_version,1,4,104}] = nif_mod_call_history(), true = erlang:delete_module(nif_mod), [] = nif_mod_call_history(), %%false= check_process_code(Pid, nif_mod), true = erlang:purge_module(nif_mod), - [{unload,1,3,103}] = nif_mod_call_history(), + [{unload,1,5,105}] = nif_mod_call_history(), true = lists:member(?MODULE, erlang:system_info(taints)), true = lists:member(nif_mod, erlang:system_info(taints)), @@ -828,7 +828,7 @@ resource_binary_do() -> -define(RT_CREATE,1). -define(RT_TAKEOVER,2). -%% Test resource takeover by module reload and upgrade +%% Test resource takeover by module upgrade resource_takeover(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -893,6 +893,7 @@ resource_takeover(Config) when is_list(Config) -> ok = forget_resource(NGX1), ?CHECK([], nif_mod_call_history()), % no dtor + {module,nif_mod} = erlang:load_module(nif_mod,ModBin), ok = nif_mod:load_nif_lib(Config, 2, [{resource_type, 0, ?RT_TAKEOVER, "resource_type_A",resource_dtor_A, ?RT_TAKEOVER}, @@ -911,7 +912,9 @@ resource_takeover(Config) when is_list(Config) -> {resource_type, 4, ?RT_CREATE, "resource_type_null_goneY",null, ?RT_CREATE} ]), - ?CHECK([{reload,2,1,201}], nif_mod_call_history()), + ?CHECK([{upgrade,2,1,201}], nif_mod_call_history()), + true = erlang:purge_module(nif_mod), + ?CHECK([{unload,1,1,106}], nif_mod_call_history()), BinA2 = read_resource(0,A2), ok = forget_resource(A2), @@ -1221,11 +1224,19 @@ threading_do(Config) -> ok = tester:load_nif_lib(Config, "basic"), ok = tester:run(), + erlang:load_module(tester,ModBin), + erlang:purge_module(tester), ok = tester:load_nif_lib(Config, "rwlock"), ok = tester:run(), + erlang:load_module(tester,ModBin), + erlang:purge_module(tester), ok = tester:load_nif_lib(Config, "tsd"), - ok = tester:run(). + ok = tester:run(), + + erlang:delete_module(tester), + erlang:purge_module(tester). + %% Test NIF message sending send(Config) when is_list(Config) -> @@ -1513,13 +1524,13 @@ send3_new_state(State, Blob) -> neg(Config) when is_list(Config) -> TmpMem = tmpmem(), {'EXIT',{badarg,_}} = (catch erlang:load_nif(badarg, 0)), - {error,{load_failed,_}} = erlang:load_nif("pink_unicorn", 0), Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "nif_mod"), {ok,nif_mod,Bin} = compile:file(File, [binary,return_errors]), {module,nif_mod} = erlang:load_module(nif_mod,Bin), + {error,{load_failed,_}} = nif_mod:load_nif_lib(Config, 0), {error,{bad_lib,_}} = nif_mod:load_nif_lib(Config, no_init), verify_tmpmem(TmpMem), ok. @@ -1640,6 +1651,7 @@ consume_timeslice(Config) when is_list(Config) -> end. consume_timeslice_test(Config) when is_list(Config) -> + ensure_lib_loaded(Config), CONTEXT_REDS = 2000, Me = self(), Go = make_ref(), @@ -1716,7 +1728,7 @@ consume_timeslice_test(Config) when is_list(Config) -> io:format("Reductions = ~p~n", [Reductions]), ok; {RedDiff, Reductions} -> - ct:fail({unexpected_reduction_count, Reductions}) + ct:fail({unexpected_reduction_count, Reductions, ExpReds}) end, none = next_msg(P), diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index f2b1ef9d24..dc6d8cea76 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> @@ -184,14 +184,6 @@ static void resource_takeover(ErlNifEnv* env, PrivData* priv) msgenv_resource_type = rt; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - PrivData* priv = (PrivData*) *priv_data; - add_call(env, priv, "reload"); - resource_takeover(env,priv); - return 0; -} - static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { PrivData* priv = (PrivData*) *old_priv_data; @@ -2094,4 +2086,4 @@ static ErlNifFunc nif_funcs[] = {"format_term_nif", 2, format_term} }; -ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c index fd8a0d0595..4e94e7901c 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.c +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <string.h> #include <stdio.h> diff --git a/erts/emulator/test/nif_SUITE_data/testcase_driver.h b/erts/emulator/test/nif_SUITE_data/testcase_driver.h index e32e63069a..feb10ecaea 100644 --- a/erts/emulator/test/nif_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/nif_SUITE_data/testcase_driver.h @@ -20,7 +20,7 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdlib.h> #include <stdio.h> diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c index 257b116322..ea4afd924d 100644 --- a/erts/emulator/test/nif_SUITE_data/tester.c +++ b/erts/emulator/test/nif_SUITE_data/tester.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <stdarg.h> @@ -53,7 +53,7 @@ void testcase_free(void *ptr) void testcase_run(TestCaseState_t *tcs); -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; } @@ -70,5 +70,5 @@ static ErlNifFunc nif_funcs[] = {"run", 0, run} }; -ERL_NIF_INIT(tester,nif_funcs,NULL,reload,NULL,NULL) +ERL_NIF_INIT(tester,nif_funcs,NULL,NULL,upgrade,NULL) diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c index 33b346aab7..786be35c9c 100644 --- a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c +++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) @@ -6,11 +6,6 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - return 0; -} - static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; @@ -34,4 +29,4 @@ static ErlNifFunc nif_funcs[] = {"nif_dec", 1, nif_dec_1} }; -ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl index 8d5bff2a48..7ac6fce234 100644 --- a/erts/emulator/test/trace_nif_SUITE.erl +++ b/erts/emulator/test/trace_nif_SUITE.erl @@ -265,10 +265,16 @@ nif_process() -> nif_process(). load_nif(Config) -> - Path = proplists:get_value(data_dir, Config), - - ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0). + case is_nif_loaded() of + true -> + ok; + false -> + Path = proplists:get_value(data_dir, Config), + ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0) + end. +is_nif_loaded() -> + false. nif() -> {"Stub0",[]}. %exit("nif/0 stub called"). diff --git a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c index 26f2420b8b..1afb5ee919 100644 --- a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c +++ b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) @@ -6,18 +6,18 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; } -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +static void unload(ErlNifEnv* env, void* priv_data) { - return 0; } -static void unload(ErlNifEnv* env, void* priv_data) +static ERL_NIF_TERM is_nif_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + return enif_make_atom(env,"true"); } static ERL_NIF_TERM nif_0(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -38,9 +38,10 @@ static ERL_NIF_TERM nif_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ErlNifFunc nif_funcs[] = { + {"is_nif_loaded", 0, is_nif_loaded}, {"nif", 0, nif_0}, {"nif", 1, nif_1} }; -ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c index a26bb33600..b68e480215 100644 --- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c +++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c @@ -18,7 +18,7 @@ * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index c5422ab2ed..30f2d831b5 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -399,19 +399,29 @@ elif [ "x$GDB" = "xdump" ]; then cmdfile="/tmp/.cerlgdb.$$" case "x$core" in x/*) - gdbcmd="$EMU_NAME ${core}" ;; *) dir=`pwd` - gdbcmd="$EMU_NAME ${dir}/${core}" + core="${dir}/${core}" ;; esac - echo "set width 0 + case `uname` in + Darwin) + echo " +thread backtrace all +quit +" > $cmdfile + exec lldb -s $cmdfile -c ${core} $EMU_NAME + ;; + *) + echo "set width 0 set height 0 set verbose off source $ROOTDIR/erts/etc/unix/etp-commands thread apply all bt " > $cmdfile - exec gdb --batch --command=$cmdfile $gdbcmd + exec gdb --batch --command=$cmdfile $EMU_NAME $core + ;; + esac fi diff --git a/erts/example/matrix_nif.c b/erts/example/matrix_nif.c index 6452084eb7..f42204d10f 100644 --- a/erts/example/matrix_nif.c +++ b/erts/example/matrix_nif.c @@ -22,7 +22,7 @@ * for matrix calculations. */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stddef.h> #include <assert.h> diff --git a/lib/asn1/c_src/asn1_erl_nif.c b/lib/asn1/c_src/asn1_erl_nif.c index 4577689fc3..b29c9a7ed3 100644 --- a/lib/asn1/c_src/asn1_erl_nif.c +++ b/lib/asn1/c_src/asn1_erl_nif.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <stdio.h> #include <string.h> -#include "erl_nif.h" +#include <erl_nif.h> /* #define ASN1_DEBUG 1 */ diff --git a/lib/common_test/doc/src/ct_telnet.xml b/lib/common_test/doc/src/ct_telnet.xml index eba3c3030d..8e85cccc99 100644 --- a/lib/common_test/doc/src/ct_telnet.xml +++ b/lib/common_test/doc/src/ct_telnet.xml @@ -337,7 +337,7 @@ <c>FullMatch</c> is the string matched by the whole regular expression, and <c>SubMatchN</c> is the string that matched subexpression number <c>N</c>. Subexpressions are denoted with - <c>(' ')</c> in the regular expression.</p> + <c>'(' ')'</c> in the regular expression.</p> <p>If a <c>Tag</c> is speciifed, the returned <c>Match</c> also includes the matched <c>Tag</c>. Otherwise, only <c>RxMatch</c> diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index cd521f04c5..f9fa80c0c7 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -31,7 +31,7 @@ #include <stdio.h> #include <string.h> -#include "erl_nif.h" +#include <erl_nif.h> #define OPENSSL_THREAD_DEFINES #include <openssl/opensslconf.h> diff --git a/lib/crypto/c_src/crypto_callback.c b/lib/crypto/c_src/crypto_callback.c index 3acbbf406b..4c23379f7f 100644 --- a/lib/crypto/c_src/crypto_callback.c +++ b/lib/crypto/c_src/crypto_callback.c @@ -22,7 +22,7 @@ #include <string.h> #include <openssl/opensslconf.h> -#include "erl_nif.h" +#include <erl_nif.h> #include "crypto_callback.h" #ifdef DEBUG diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 91256fa6a2..d8bdac24e3 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -110,27 +110,30 @@ result(Response = {{_, 300, _}, _, _}, redirect(Response, Request); result(Response = {{_, Code, _}, _, _}, + Request = #request{settings = + #http_options{autoredirect = true}, + method = post}) when (Code =:= 301) orelse + (Code =:= 302) orelse + (Code =:= 303) -> + redirect(Response, Request#request{method = get}); +result(Response = {{_, Code, _}, _, _}, + Request = #request{settings = + #http_options{autoredirect = true}, + method = post}) when (Code =:= 307) -> + redirect(Response, Request); +result(Response = {{_, Code, _}, _, _}, Request = #request{settings = #http_options{autoredirect = true}, - method = head}) when (Code =:= 301) orelse + method = Method}) when (Code =:= 301) orelse (Code =:= 302) orelse (Code =:= 303) orelse (Code =:= 307) -> - redirect(Response, Request); -result(Response = {{_, Code, _}, _, _}, - Request = #request{settings = - #http_options{autoredirect = true}, - method = get}) when (Code =:= 301) orelse - (Code =:= 302) orelse - (Code =:= 303) orelse - (Code =:= 307) -> - redirect(Response, Request); -result(Response = {{_, 303, _}, _, _}, - Request = #request{settings = - #http_options{autoredirect = true}, - method = post}) -> - redirect(Response, Request#request{method = get}); - + case lists:member(Method, [get, head, options, trace]) of + true -> + redirect(Response, Request); + false -> + transparent(Response, Request) + end; result(Response = {{_,503,_}, _, _}, Request) -> status_service_unavailable(Response, Request); diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 932567ec55..57da82c6ad 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -500,10 +500,11 @@ redirect_multiple_choises(Config) when is_list(Config) -> httpc:request(get, {URL300, []}, [{autoredirect, false}], []). %%------------------------------------------------------------------------- redirect_moved_permanently() -> - [{doc, "If the 301 status code is received in response to a request other " - "than GET or HEAD, the user agent MUST NOT automatically redirect the request " - "unless it can be confirmed by the user, since this might change " - "the conditions under which the request was issued."}]. + [{doc, "The server SHOULD generate a Location header field in the response " + "containing a preferred URI reference for the new permanent URI. The user " + "agent MAY use the Location field value for automatic redirection. The server's " + "response payload usually contains a short hypertext note with a " + "hyperlink to the new URI(s)."}]. redirect_moved_permanently(Config) when is_list(Config) -> URL301 = url(group_name(Config), "/301.html", Config), @@ -514,15 +515,16 @@ redirect_moved_permanently(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], []}} = httpc:request(head, {URL301, []}, [], []), - {ok, {{_,301,_}, [_ | _], [_|_]}} + {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL301, [],"text/plain", "foobar"}, [], []). %%------------------------------------------------------------------------- redirect_found() -> - [{doc," If the 302 status code is received in response to a request other " - "than GET or HEAD, the user agent MUST NOT automatically redirect the " - "request unless it can be confirmed by the user, since this might change " - "the conditions under which the request was issued."}]. + [{doc, "The server SHOULD generate a Location header field in the response " + "containing a URI reference for the different URI. The user agent MAY " + "use the Location field value for automatic redirection. The server's " + "response payload usually contains a short hypertext note with a " + "hyperlink to the different URI(s)."}]. redirect_found(Config) when is_list(Config) -> URL302 = url(group_name(Config), "/302.html", Config), @@ -533,14 +535,14 @@ redirect_found(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], []}} = httpc:request(head, {URL302, []}, [], []), - {ok, {{_,302,_}, [_ | _], [_|_]}} + {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL302, [],"text/plain", "foobar"}, [], []). %%------------------------------------------------------------------------- redirect_see_other() -> [{doc, "The different URI SHOULD be given by the Location field in the response. " "Unless the request method was HEAD, the entity of the response SHOULD contain a short " - "hypertext note with a hyperlink to the new URI(s). "}]. + "hypertext note with a hyperlink to the new URI(s)."}]. redirect_see_other(Config) when is_list(Config) -> URL303 = url(group_name(Config), "/303.html", Config), @@ -556,10 +558,11 @@ redirect_see_other(Config) when is_list(Config) -> [], []). %%------------------------------------------------------------------------- redirect_temporary_redirect() -> - [{doc," If the 307 status code is received in response to a request other " - "than GET or HEAD, the user agent MUST NOT automatically redirect the request " - "unless it can be confirmed by the user, since this might change " - "the conditions under which the request was issued."}]. + [{doc, "The server SHOULD generate a Location header field in the response " + "containing a URI reference for the different URI. The user agent MAY " + "use the Location field value for automatic redirection. The server's " + "response payload usually contains a short hypertext note with a " + "hyperlink to the different URI(s)."}]. redirect_temporary_redirect(Config) when is_list(Config) -> URL307 = url(group_name(Config), "/307.html", Config), @@ -570,7 +573,7 @@ redirect_temporary_redirect(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], []}} = httpc:request(head, {URL307, []}, [], []), - {ok, {{_,307,_}, [_ | _], [_|_]}} + {ok, {{_,200,_}, [_ | _], [_|_]}} = httpc:request(post, {URL307, [],"text/plain", "foobar"}, [], []). diff --git a/lib/kernel/doc/src/code.xml b/lib/kernel/doc/src/code.xml index 4db377bcde..3143cdc825 100644 --- a/lib/kernel/doc/src/code.xml +++ b/lib/kernel/doc/src/code.xml @@ -382,9 +382,14 @@ zip:create("mnesia-4.4.7.ez", <name name="add_pathsa" arity="1"/> <fsummary>Add directories to the beginning of the code path.</fsummary> <desc> - <p>Adds the directories in <c><anno>Dirs</anno></c> to the beginning of - the code path. If a <c><anno>Dir</anno></c> exists, it is removed - from the old position in the code path.</p> + <p>Traverses <c><anno>Dirs</anno></c> and adds + each <c><anno>Dir</anno></c> to the beginning of the code + path. This means that the order of <c><anno>Dirs</anno></c> + is reversed in the resulting code path. For example, if you + add <c>[Dir1,Dir2]</c>, the resulting path will + be <c>[Dir2,Dir1|OldCodePath]</c>.</p> + <p>If a <c><anno>Dir</anno></c> already exists in the code + path, it is removed from the old position.</p> <p>Always returns <c>ok</c>, regardless of the validity of each individual <c><anno>Dir</anno></c>.</p> </desc> diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 58b601e456..1971df9038 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -397,25 +397,36 @@ write_file(Name, Bin) -> Modes :: [mode()], Reason :: posix() | badarg | terminated | system_limit. -write_file(Name, Bin, ModeList) when is_list(ModeList) -> - case make_binary(Bin) of - B when is_binary(B) -> - case open(Name, [binary, write | - lists:delete(binary, - lists:delete(write, ModeList))]) of - {ok, Handle} -> - case write(Handle, B) of - ok -> - close(Handle); - E1 -> - _ = close(Handle), - E1 - end; - E2 -> - E2 - end; - E3 -> - E3 +write_file(Name, IOData, ModeList) when is_list(ModeList) -> + case lists:member(raw, ModeList) of + true -> + %% For backwards compatibility of error messages + try iolist_size(IOData) of + _Size -> do_write_file(Name, IOData, ModeList) + catch + error:Error -> {error, Error} + end; + false -> + case make_binary(IOData) of + Bin when is_binary(Bin) -> + do_write_file(Name, Bin, ModeList); + Error -> + Error + end + end. + +do_write_file(Name, IOData, ModeList) -> + case open(Name, [binary, write | ModeList]) of + {ok, Handle} -> + case write(Handle, IOData) of + ok -> + close(Handle); + E1 -> + _ = close(Handle), + E1 + end; + E2 -> + E2 end. %% Obsolete, undocumented, local node only, don't use!. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index d16e200cb3..82cf73cbda 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -18,9 +18,9 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"5\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* + [{<<"5\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* {<<"4\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-18.* %% Down to - max one major revision back - [{<<"5\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* + [{<<"5\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* {<<"4\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-18.* }. diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 5f049c6f99..c37d114a58 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -493,22 +493,13 @@ read_write_file(Config) when is_list(Config) -> %% Try writing and reading back some term SomeTerm = {"This term",{will,be},[written,$t,$o],1,file,[]}, - ok = ?FILE_MODULE:write_file(Name,term_to_binary(SomeTerm)), - {ok,Bin1} = ?FILE_MODULE:read_file(Name), - SomeTerm = binary_to_term(Bin1), + Bin1 = term_to_binary(SomeTerm), + ok = do_read_write_file(Name, Bin1), %% Try a "null" term NullTerm = [], - ok = ?FILE_MODULE:write_file(Name,term_to_binary(NullTerm)), - {ok,Bin2} = ?FILE_MODULE:read_file(Name), - NullTerm = binary_to_term(Bin2), - - %% Try some "complicated" types - BigNum = 123456789012345678901234567890, - ComplTerm = {self(),make_ref(),BigNum,3.14159}, - ok = ?FILE_MODULE:write_file(Name,term_to_binary(ComplTerm)), - {ok,Bin3} = ?FILE_MODULE:read_file(Name), - ComplTerm = binary_to_term(Bin3), + Bin2 = term_to_binary(NullTerm), + ok = do_read_write_file(Name, Bin2), %% Try reading a nonexistent file Name2 = filename:join(RootDir, @@ -519,25 +510,42 @@ read_write_file(Config) when is_list(Config) -> {error, enoent} = ?FILE_MODULE:read_file(''), %% Try writing to a bad filename - {error, enoent} = - ?FILE_MODULE:write_file("",term_to_binary(NullTerm)), + {error, enoent} = do_read_write_file("", Bin2), %% Try writing something else than a binary - {error, badarg} = ?FILE_MODULE:write_file(Name,{1,2,3}), - {error, badarg} = ?FILE_MODULE:write_file(Name,self()), + {error, badarg} = do_read_write_file(Name, {1,2,3}), + {error, badarg} = do_read_write_file(Name, self()), %% Some non-term binaries - ok = ?FILE_MODULE:write_file(Name,[]), - {ok,Bin4} = ?FILE_MODULE:read_file(Name), - 0 = byte_size(Bin4), + ok = do_read_write_file(Name, []), - ok = ?FILE_MODULE:write_file(Name,[Bin1,[],[[Bin2]]]), - {ok,Bin5} = ?FILE_MODULE:read_file(Name), - {Bin1,Bin2} = split_binary(Bin5,byte_size(Bin1)), + %% Write some iolists + ok = do_read_write_file(Name, [Bin1,[],[[Bin2]]]), + ok = do_read_write_file(Name, ["string",<<"binary">>]), + ok = do_read_write_file(Name, "pure string"), [] = flush(), ok. +do_read_write_file(Name, Data) -> + case ?FILE_MODULE:write_file(Name, Data) of + ok -> + BinData = iolist_to_binary(Data), + {ok,BinData} = ?FILE_MODULE:read_file(Name), + + ok = ?FILE_MODULE:write_file(Name, Data, []), + {ok,BinData} = ?FILE_MODULE:read_file(Name), + + ok = ?FILE_MODULE:write_file(Name, Data, [raw]), + {ok,BinData} = ?FILE_MODULE:read_file(Name), + + ok; + {error,_}=Res -> + Res = ?FILE_MODULE:write_file(Name, Data, []), + Res = ?FILE_MODULE:write_file(Name, Data, [raw]), + Res + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c index 2990ddda41..b91dca61d4 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c +++ b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> diff --git a/lib/observer/doc/src/ttb.xml b/lib/observer/doc/src/ttb.xml index 42b0fa1d8a..7cd15e15d3 100644 --- a/lib/observer/doc/src/ttb.xml +++ b/lib/observer/doc/src/ttb.xml @@ -114,7 +114,8 @@ ttb:p(all, call).</input></pre> <v>Opt = {file,Client} | {handler, FormatHandler} | {process_info,PI} | shell | {shell, ShellSpec} | {timer, TimerSpec} | {overload_check, {MSec, Module, Function}} | - {flush, MSec} | resume | {resume, FetchTimeout}</v> + {flush, MSec} | resume | {resume, FetchTimeout} | + {queue_size, QueueSize}</v> <v>TimerSpec = MSec | {MSec, StopOpts}</v> <v>MSec = FetchTimeout = integer()</v> <v>Module = Function = atom() </v> @@ -126,6 +127,7 @@ ttb:p(all, call).</input></pre> <v>FormatHandler = See format/2</v> <v>PI = true | false </v> <v>ShellSpec = true | false | only</v> + <v>QueueSize = non_neg_integer()</v> </type> <desc> <p>Starts a file trace port on all specified nodes @@ -147,6 +149,18 @@ ttb:p(all, call).</input></pre> <c>Client</c> must be <c>{local, File}</c>. All trace information is then sent to the trace control node where it is written to file.</p></item> + <tag><c>queue_size</c></tag> + <item><p>When tracing to shell or <c>{local,File}</c>, an ip + trace driver is used internally. The ip trace driver has a + queue of maximum <c>QueueSize</c> messages waiting to be + delivered. If the driver cannot deliver messages as fast as + they are produced, the queue size might be exceeded and + messages are dropped. This parameter is optional, and is + only useful if many <c>{drop,N}</c> trace messages are + received by the trace handler. It has no meaning if shell + or <c>{local,File}</c> is not used. See + <seealso marker="runtime_tools:dbg#trace_port/2">dbg:trace_port/2</seealso> + for more information about the ip trace driver.</p></item> <tag><c>process_info</c></tag> <item><p>Indicates if process information is to be collected. If <c>PI = true</c> (which is diff --git a/lib/observer/src/ttb.erl b/lib/observer/src/ttb.erl index ac6c4572eb..87a50e046b 100644 --- a/lib/observer/src/ttb.erl +++ b/lib/observer/src/ttb.erl @@ -78,6 +78,11 @@ do_tracer(Nodes0,PI,Client,Traci) -> do_tracer(Clients,PI,Traci) -> Shell = proplists:get_value(shell, Traci, false), + IpPortSpec = + case proplists:get_value(queue_size, Traci) of + undefined -> 0; + QS -> {0,QS} + end, DefShell = fun(Trace) -> dbg:dhandler(Trace, standard_io) end, {ClientSucc,Succ} = lists:foldl( @@ -98,7 +103,7 @@ do_tracer(Clients,PI,Traci) -> [_,H] = string:tokens(atom_to_list(N),"@"), H end, - case catch dbg:tracer(N,port,dbg:trace_port(ip,0)) of + case catch dbg:tracer(N,port,dbg:trace_port(ip,IpPortSpec)) of {ok,N} -> {ok,Port} = dbg:trace_port_control(N,get_listen_port), {ok,T} = dbg:get_tracer(N), @@ -160,6 +165,8 @@ opt([{resume,MSec}|O],{PI,Client,Traci}) -> opt(O,{PI,Client,[{resume, {true, MSec}}|Traci]}); opt([{flush,MSec}|O],{PI,Client,Traci}) -> opt(O,{PI,Client,[{flush, MSec}|Traci]}); +opt([{queue_size,QueueSize}|O],{PI,Client,Traci}) -> + opt(O,{PI,Client,[{queue_size,QueueSize}|Traci]}); opt([],Opt) -> ensure_opt(Opt). diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl index 15d42a4d9c..602e47404d 100644 --- a/lib/parsetools/src/leex.erl +++ b/lib/parsetools/src/leex.erl @@ -1586,6 +1586,8 @@ out_dfa_graph(St, DFA, DF) -> case file:open(St#leex.gfile, [write]) of {ok,Gfile} -> try + %% Set the same encoding as infile: + set_encoding(St, Gfile), io:fwrite(Gfile, "digraph DFA {~n", []), out_dfa_states(Gfile, DFA, DF), out_dfa_edges(Gfile, DFA), @@ -1621,7 +1623,7 @@ out_dfa_edges(File, DFA) -> foreach(fun (T) -> Crs = orddict:fetch(T, Tdict), Edgelab = dfa_edgelabel(Crs), - io:fwrite(File, " ~b -> ~b [label=\"~s\"];~n", + io:fwrite(File, " ~b -> ~b [label=\"~ts\"];~n", [S,T,Edgelab]) end, sort(orddict:fetch_keys(Tdict))) end, DFA). diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 949ef3c36e..84f9c996ac 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -408,12 +408,12 @@ unicode(Config) when is_list(Config) -> Ts = [{unicode_1, <<"%% -*- coding: utf-8 -*-\n" "Definitions.\n" - "RTLarrow = (←)\n" + "RTLarrow = (â)\n" "Rules.\n" - "{RTLarrow} : {token,{'<-',TokenLine}}.\n" + "{RTLarrow} : {token,{\"â\",TokenLine}}.\n" "Erlang code.\n" "-export([t/0]).\n" - "t() -> {ok, [{'<-', 1}], 1} = string(\"←\"), ok.">>, + "t() -> {ok, [{\"â\", 1}], 1} = string(\"â\"), ok.">>, default, ok}], @@ -1137,7 +1137,7 @@ run_test(Config, Def, Pre) -> XrlFile = filename:join(DataDir, DefFile), ErlFile = filename:join(DataDir, Filename), Opts = [return, warn_unused_vars,{outdir,DataDir}], - ok = file:write_file(XrlFile, Def, [{encoding, unicode}]), + ok = file:write_file(XrlFile, Def), LOpts = [return, {report, false} | case Pre of default -> diff --git a/lib/public_key/doc/src/public_key_app.xml b/lib/public_key/doc/src/public_key_app.xml index 1f87932b6c..923a9f1dfb 100644 --- a/lib/public_key/doc/src/public_key_app.xml +++ b/lib/public_key/doc/src/public_key_app.xml @@ -61,7 +61,7 @@ <section> <title>DEPENDENCIES</title> <p>The <c>public_key</c> application uses the - Crypto application to preform cryptographic operations and the + Crypto application to perform cryptographic operations and the ASN-1 application to handle PKIX-ASN-1 specifications, hence these applications must be loaded for the <c>public_key</c> application to work. In an embedded environment this means they must be started with @@ -72,7 +72,7 @@ <section> <title>ERROR LOGGER AND EVENT HANDLERS</title> <p> The <c>public_key</c> application is a library application - and does not use the error logger. The functions will either sucssed + and does not use the error logger. The functions will either succeed or fail with a runtime error. </p> </section> diff --git a/lib/runtime_tools/c_src/dyntrace.c b/lib/runtime_tools/c_src/dyntrace.c index 3d2de9c21c..b589f5ef07 100644 --- a/lib/runtime_tools/c_src/dyntrace.c +++ b/lib/runtime_tools/c_src/dyntrace.c @@ -24,7 +24,7 @@ -#include "erl_nif.h" +#include <erl_nif.h> #include "config.h" #include "sys.h" #include "dtrace-wrapper.h" diff --git a/lib/runtime_tools/doc/src/dbg.xml b/lib/runtime_tools/doc/src/dbg.xml index db04bfdf7b..95f74d4607 100644 --- a/lib/runtime_tools/doc/src/dbg.xml +++ b/lib/runtime_tools/doc/src/dbg.xml @@ -954,7 +954,7 @@ Error: fun containing local erlang function calls ('is_atomm' called in guard)\ as the tuple <c>{drop, N}</c> where <c>N</c> is the number of consecutive messages dropped. In case of heavy tracing, drop's are likely to occur, and they surely occur if no client is reading the trace - messages.</p> + messages. The default value of <c>QueSize</c> is 200.</p> </desc> </func> <func> diff --git a/lib/runtime_tools/src/dbg.erl b/lib/runtime_tools/src/dbg.erl index c0d4665bda..f17aa528ed 100644 --- a/lib/runtime_tools/src/dbg.erl +++ b/lib/runtime_tools/src/dbg.erl @@ -427,7 +427,7 @@ trace_port(file, Filename) -> trace_port1(file, Filename, nowrap); trace_port(ip, Portno) when is_integer(Portno) -> - trace_port(ip,{Portno,50}); + trace_port(ip,{Portno,200}); trace_port(ip, {Portno, Qsiz}) when is_integer(Portno), is_integer(Qsiz) -> fun() -> diff --git a/lib/runtime_tools/test/dbg_SUITE_data/dbg_SUITE.c b/lib/runtime_tools/test/dbg_SUITE_data/dbg_SUITE.c index ecdee7e3a2..ad90af4da0 100644 --- a/lib/runtime_tools/test/dbg_SUITE_data/dbg_SUITE.c +++ b/lib/runtime_tools/test/dbg_SUITE_data/dbg_SUITE.c @@ -18,7 +18,7 @@ * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> diff --git a/lib/sasl/doc/src/script.xml b/lib/sasl/doc/src/script.xml index 8ed132354d..b40ff28179 100644 --- a/lib/sasl/doc/src/script.xml +++ b/lib/sasl/doc/src/script.xml @@ -88,7 +88,7 @@ follows:</p> <list type="bulleted"> <item><c>-pa Dir1 Dir2 ... DirN</c> adds the directories - <c>Dir1, Dir2, ..., DirN</c> to the front of the initial + <c>DirN, DirN-1, ..., Dir2, Dir1</c> to the front of the initial load path.</item> <item><c>-pz Dir1 Dir2 ... DirN</c> adds the directories <c>Dir1, Dir2, ..., DirN</c> to the end of the initial diff --git a/lib/ssh/src/ssh_cli.erl b/lib/ssh/src/ssh_cli.erl index 74cd2e081a..8af0ecc5f9 100644 --- a/lib/ssh/src/ssh_cli.erl +++ b/lib/ssh/src/ssh_cli.erl @@ -208,8 +208,15 @@ handle_msg({Group, Req}, #state{group = Group, buf = Buf, pty = Pty, write_chars(ConnectionHandler, ChannelId, Chars), {ok, State#state{buf = NewBuf}}; -handle_msg({'EXIT', Group, _Reason}, #state{group = Group, - channel = ChannelId} = State) -> +handle_msg({'EXIT', Group, Reason}, #state{group = Group, + cm = ConnectionHandler, + channel = ChannelId} = State) -> + Status = case Reason of + normal -> 0; + _ -> -1 + end, + ssh_connection:exit_status(ConnectionHandler, ChannelId, Status), + ssh_connection:send_eof(ConnectionHandler, ChannelId), {stop, ChannelId, State}; handle_msg(_, State) -> diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index d52d453007..51e0d5196b 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -67,7 +67,8 @@ shell_unicode_string/1, ssh_info_print/1, key_callback/1, - key_callback_options/1 + key_callback_options/1, + shell_exit_status/1 ]). %%% Common test callbacks @@ -106,7 +107,8 @@ all() -> multi_daemon_opt_fd, packet_size_zero, ssh_info_print, - {group, login_bad_pwd_no_retry} + {group, login_bad_pwd_no_retry}, + shell_exit_status ]. groups() -> @@ -1167,6 +1169,33 @@ login_bad_pwd_no_retry(Config, AuthMethods) -> end end. + +%%---------------------------------------------------------------------------- +%%% Test that when shell REPL exit with reason normal client receives status 0 +shell_exit_status(Config) when is_list(Config) -> + process_flag(trap_exit, true), + SystemDir = proplists:get_value(data_dir, Config), + UserDir = proplists:get_value(priv_dir, Config), + + ShellFun = fun (_User) -> spawn(fun() -> ok end) end, + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]}, + {shell, ShellFun}, + {failfun, fun ssh_test_lib:failfun/2}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user_dir, UserDir}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}]), + + {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:shell(ConnectionRef, ChannelId), + ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId), + ssh:stop_daemon(Pid). + + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 212b99c695..09e707ad07 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.3.2 +SSH_VSN = 4.3.3 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/stdlib/src/edlin_expand.erl b/lib/stdlib/src/edlin_expand.erl index ec64470461..5f821caef0 100644 --- a/lib/stdlib/src/edlin_expand.erl +++ b/lib/stdlib/src/edlin_expand.erl @@ -118,7 +118,7 @@ format_col([A|T], Width, Len, Acc0) -> {H1, _} -> H1; H2 -> H2 end, - Acc = [io_lib:format("~-*s", [Width,H]) | Acc0], + Acc = [io_lib:format("~-*ts", [Width,H]) | Acc0], format_col(T, Width, Len+Width, Acc); format_col([], _, _, Acc) -> lists:reverse(Acc, "\n"). diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 9877662743..e917b7ea1f 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -18,9 +18,9 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* + [{<<"3\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* {<<"2\\.[5-8](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-18.* %% Down to - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* + [{<<"3\\.[0-1](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.* {<<"2\\.[5-8](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-18.* }. diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index c81e72689c..1cd65fbf18 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -1087,6 +1087,10 @@ wait_dynamic_children(#child{restart_type=RType} = Child, Pids, Sz, wait_dynamic_children(Child, ?SETS:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, {shutdown, _}} -> + wait_dynamic_children(Child, ?SETS:del_element(Pid, Pids), Sz-1, + TRef, EStack); + {'DOWN', _MRef, process, Pid, normal} when RType =/= permanent -> wait_dynamic_children(Child, ?SETS:del_element(Pid, Pids), Sz-1, TRef, EStack); diff --git a/lib/stdlib/test/edlin_expand_SUITE.erl b/lib/stdlib/test/edlin_expand_SUITE.erl index ccffa2e244..718d91c6a3 100644 --- a/lib/stdlib/test/edlin_expand_SUITE.erl +++ b/lib/stdlib/test/edlin_expand_SUITE.erl @@ -21,7 +21,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_testcase/2, end_per_testcase/2, init_per_group/2,end_per_group/2]). --export([normal/1, quoted_fun/1, quoted_module/1, quoted_both/1]). +-export([normal/1, quoted_fun/1, quoted_module/1, quoted_both/1, erl_1152/1]). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +36,7 @@ suite() -> {timetrap,{minutes,1}}]. all() -> - [normal, quoted_fun, quoted_module, quoted_both]. + [normal, quoted_fun, quoted_module, quoted_both, erl_1152]. groups() -> []. @@ -149,5 +149,12 @@ quoted_both(Config) when is_list(Config) -> {yes,"weird-fun-name'()",[]} = do_expand("'ExpandTestCaps1':'#"), ok. +erl_1152(Config) when is_list(Config) -> + "\n"++"foo"++" "++[1089]++_ = do_format(["foo",[1089]]), + ok. + do_expand(String) -> edlin_expand:expand(lists:reverse(String)). + +do_format(StringList) -> + lists:flatten(edlin_expand:format_matches(StringList)). diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index b02d17bdb6..e726a29c9e 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -218,7 +218,7 @@ memory_check_summary(_Config) -> receive {get_failed_memchecks, FailedMemchecks} -> ok end, io:format("Failed memchecks: ~p\n",[FailedMemchecks]), NoFailedMemchecks = length(FailedMemchecks), - if NoFailedMemchecks > 3 -> + if NoFailedMemchecks > 1 -> ct:fail("Too many failed (~p) memchecks", [NoFailedMemchecks]); true -> ok @@ -591,12 +591,6 @@ select_fail_do(Opts) -> -define(S(T),ets:info(T,memory)). --define(TAB_STRUCT_SZ, erts_debug:get_internal_state('DbTable_words')). -%%-define(NORMAL_TAB_STRUCT_SZ, 26). %% SunOS5.8, 32-bit, non smp, private heap -%% -%% The hardcoded expected memory sizes (in words) are the ones we expect on: -%% SunOS5.8, 32-bit, non smp, private heap -%% %% Whitebox test of ets:info(X, memory). memory(Config) when is_list(Config) -> @@ -607,11 +601,11 @@ memory(Config) when is_list(Config) -> memory_do(Opts) -> L = [T1,T2,T3,T4] = fill_sets_int(1000,Opts), XR1 = case mem_mode(T1) of - {normal,_} -> {13836,13046,13046,13052}; %{13862,13072,13072,13078}; - {compressed,4} -> {11041,10251,10251,10252}; %{11067,10277,10277,10278}; - {compressed,8} -> {10050,9260,9260,9260} %{10076,9286,9286,9286} + {normal,_} -> {13836, 15346, 15346, 15346+6}; + {compressed,4} -> {11041, 12551, 12551, 12551+1}; + {compressed,8} -> {10050, 11560, 11560, 11560} end, - XRes1 = adjust_xmem(L, XR1), + XRes1 = adjust_xmem(L, XR1, 1), Res1 = {?S(T1),?S(T2),?S(T3),?S(T4)}, lists:foreach(fun(T) -> Before = ets:info(T,size), @@ -623,11 +617,11 @@ memory_do(Opts) -> end, L), XR2 = case mem_mode(T1) of - {normal,_} -> {13826,13037,13028,13034}; %{13852,13063,13054,13060}; - {compressed,4} -> {11031,10242,10233,10234}; %{11057,10268,10259,10260}; - {compressed,8} -> {10040,9251,9242,9242} %10066,9277,9268,9268} + {normal,_} -> {13826, 15337, 15337-9, 15337-3}; + {compressed,4} -> {11031, 12542, 12542-9, 12542-8}; + {compressed,8} -> {10040, 11551, 11551-9, 11551-9} end, - XRes2 = adjust_xmem(L, XR2), + XRes2 = adjust_xmem(L, XR2, 1), Res2 = {?S(T1),?S(T2),?S(T3),?S(T4)}, lists:foreach(fun(T) -> Before = ets:info(T,size), @@ -639,17 +633,17 @@ memory_do(Opts) -> end, L), XR3 = case mem_mode(T1) of - {normal,_} -> {13816,13028,13010,13016}; %{13842,13054,13036,13042}; - {compressed,4} -> {11021,10233,10215,10216}; %{11047,10259,10241,10242}; - {compressed,8} -> {10030,9242,9224,9224} %{10056,9268,9250,9250} + {normal,_} -> {13816, 15328, 15328-18, 15328-12}; + {compressed,4} -> {11021, 12533, 12533-18, 12533-17}; + {compressed,8} -> {10030, 11542, 11542-18, 11542-18} end, - XRes3 = adjust_xmem(L, XR3), + XRes3 = adjust_xmem(L, XR3, 1), Res3 = {?S(T1),?S(T2),?S(T3),?S(T4)}, lists:foreach(fun(T) -> ets:delete_all_objects(T) end, L), - XRes4 = adjust_xmem(L, {50,260,260,260}), %{76,286,286,286}), + XRes4 = adjust_xmem(L, {50, 256, 256, 256}, 0), Res4 = {?S(T1),?S(T2),?S(T3),?S(T4)}, lists:foreach(fun(T) -> ets:delete(T) @@ -660,7 +654,7 @@ memory_do(Opts) -> ets:select_delete(T,[{'_',[],[true]}]) end, L2), - XRes5 = adjust_xmem(L2, {50,260,260,260}), %{76,286,286,286}), + XRes5 = adjust_xmem(L2, {50, 256, 256, 256}, 0), Res5 = {?S(T11),?S(T12),?S(T13),?S(T14)}, io:format("XRes1 = ~p~n" " Res1 = ~p~n~n" @@ -698,15 +692,15 @@ chk_normal_tab_struct_size() -> erlang:system_info(smp_support), erlang:system_info(heap_type)}, io:format("System = ~p~n", [System]), - io:format("?TAB_STRUCT_SZ=~p~n", [?TAB_STRUCT_SZ]), ok. -adjust_xmem([_T1,_T2,_T3,_T4], {A0,B0,C0,D0} = _Mem0) -> +adjust_xmem([_T1,_T2,_T3,_T4], {A0,B0,C0,D0} = _Mem0, EstCnt) -> %% Adjust for 64-bit, smp, and os: %% Table struct size may differ. - TabDiff = ?TAB_STRUCT_SZ, - {A0+TabDiff, B0+TabDiff, C0+TabDiff, D0+TabDiff}. + {TabSz, EstSz} = erts_debug:get_internal_state('DbTable_words'), + HTabSz = TabSz + EstCnt*EstSz, + {A0+TabSz, B0+HTabSz, C0+HTabSz, D0+HTabSz}. %% Misc. whitebox tests t_whitebox(Config) when is_list(Config) -> @@ -1908,7 +1902,7 @@ evil_counter(I,Opts) -> end, Start = Start0 + rand:uniform(100000), ets:insert(T, {dracula,Start}), - Iter = 40000, + Iter = 40000 div syrup_factor(), End = Start + Iter, End = evil_counter_1(Iter, T), ets:delete(T). @@ -3302,7 +3296,8 @@ evil_delete_owner(Name, Flags, Data, Fix) -> exit_large_table_owner(Config) when is_list(Config) -> %%Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 500000)], - FEData = fun(Do) -> repeat_while(fun(500000) -> {false,ok}; + Laps = 500000 div syrup_factor(), + FEData = fun(Do) -> repeat_while(fun(I) when I =:= Laps -> {false,ok}; (I) -> Do({erlang:phash2(I, 16#ffffff),I}), {true, I+1} end, 1) @@ -3318,7 +3313,8 @@ exit_large_table_owner_do(Opts,{FEData,Config}) -> exit_many_large_table_owner(Config) when is_list(Config) -> ct:timetrap({minutes,30}), %% valgrind needs a lot %%Data = [{erlang:phash2(I, 16#ffffff),I} || I <- lists:seq(1, 500000)], - FEData = fun(Do) -> repeat_while(fun(500000) -> {false,ok}; + Laps = 500000 div syrup_factor(), + FEData = fun(Do) -> repeat_while(fun(I) when I =:= Laps -> {false,ok}; (I) -> Do({erlang:phash2(I, 16#ffffff),I}), {true, I+1} end, 1) @@ -4271,7 +4267,8 @@ heavy_lookup_element_do(Opts) -> Tab = ets_new(foobar_table, [set, protected, {keypos, 2} | Opts]), ok = fill_tab2(Tab, 0, 7000), %% lookup ALL elements 50 times - _ = [do_lookup_element(Tab, 6999, 1) || _ <- lists:seq(1, 50)], + Laps = 50 div syrup_factor(), + _ = [do_lookup_element(Tab, 6999, 1) || _ <- lists:seq(1, Laps)], true = ets:delete(Tab), verify_etsmem(EtsMem). @@ -4295,6 +4292,7 @@ heavy_concurrent(Config) when is_list(Config) -> do_heavy_concurrent(Opts) -> Size = 10000, + Laps = 10000 div syrup_factor(), EtsMem = etsmem(), Tab = ets_new(blupp, [set, public, {keypos, 2} | Opts]), ok = fill_tab2(Tab, 0, Size), @@ -4302,7 +4300,7 @@ do_heavy_concurrent(Opts) -> fun (N) -> my_spawn_link( fun () -> - do_heavy_concurrent_proc(Tab, Size, N) + do_heavy_concurrent_proc(Tab, Laps, N) end) end, lists:seq(1, 500)), @@ -5363,12 +5361,12 @@ verify_table_load(T) -> Stats = ets:info(T,stats), {Buckets,AvgLen,StdDev,ExpSD,_MinLen,_MaxLen,_} = Stats, ok = if - AvgLen > 7 -> + AvgLen > 1.2 -> io:format("Table overloaded: Stats=~p\n~p\n", [Stats, ets:info(T)]), false; - Buckets>256, AvgLen < 6 -> + Buckets>256, AvgLen < 0.47 -> io:format("Table underloaded: Stats=~p\n~p\n", [Stats, ets:info(T)]), false; @@ -5442,7 +5440,8 @@ smp_select_delete(Config) when is_list(Config) -> Eq+1 end, 0, TotCnts), - verify_table_load(T), + %% May fail as select_delete does not shrink table (enough) + %%verify_table_load(T), LeftInTab = ets:select_delete(T, [{{'$1','$1'}, [], [true]}]), 0 = ets:info(T,size), false = ets:info(T,fixed), @@ -5718,27 +5717,45 @@ etsmem() -> {Bl0+Bl,BlSz0+BlSz} end, {0,0}, CS) end}, - {Mem,AllTabs}. + {Mem,AllTabs, erts_debug:get_internal_state('DbTable_meta')}. -verify_etsmem({MemInfo,AllTabs}) -> +verify_etsmem(EtsMem) -> wait_for_test_procs(), + verify_etsmem(EtsMem, false). + +verify_etsmem({MemInfo,AllTabs,MetaState}=EtsMem, Adjusted) -> case etsmem() of - {MemInfo,_} -> + {MemInfo,_,_} -> io:format("Ets mem info: ~p", [MemInfo]), case MemInfo of {ErlMem,EtsAlloc} when ErlMem == notsup; EtsAlloc == undefined -> %% Use 'erl +Mea max' to do more complete memory leak testing. {comment,"Incomplete or no mem leak testing"}; _ -> - ok + case Adjusted of + true -> + {comment, "Meta state adjusted"}; + false -> + ok + end end; - {MemInfo2, AllTabs2} -> + + {MemInfo2, AllTabs2, MetaState2} -> io:format("Expected: ~p", [MemInfo]), io:format("Actual: ~p", [MemInfo2]), io:format("Changed tables before: ~p\n",[AllTabs -- AllTabs2]), io:format("Changed tables after: ~p\n", [AllTabs2 -- AllTabs]), - ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')}, - {comment, "Failed memory check"} + io:format("Meta state before: ~p\n", [MetaState]), + io:format("Meta state after: ~p\n", [MetaState2]), + case {MetaState =:= MetaState2, Adjusted} of + {false, false} -> + io:format("Adjust meta state and retry...\n\n",[]), + {ok,ok} = erts_debug:set_internal_state('DbTable_meta', MetaState), + verify_etsmem(EtsMem, true); + _ -> + ets_test_spawn_logger ! {failed_memcheck, get('__ETS_TEST_CASE__')}, + {comment, "Failed memory check"} + end end. @@ -6261,3 +6278,9 @@ do_tc(Do, Report) -> T2 = erlang:monotonic_time(), Elapsed = erlang:convert_time_unit(T2 - T1, native, milli_seconds), Report(Elapsed). + +syrup_factor() -> + case erlang:system_info(build_type) of + valgrind -> 20; + _ -> 1 + end. diff --git a/system/doc/tutorial/complex6_nif.c b/system/doc/tutorial/complex6_nif.c index b656ed43ce..f6c06e94f4 100644 --- a/system/doc/tutorial/complex6_nif.c +++ b/system/doc/tutorial/complex6_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> extern int foo(int x); extern int bar(int y); |