diff options
67 files changed, 1681 insertions, 574 deletions
diff --git a/OTP_VERSION b/OTP_VERSION index 8b74066182..758704512b 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -22.0.3 +22.0.4 diff --git a/erts/doc/src/alt_dist.xml b/erts/doc/src/alt_dist.xml index 7c997cae20..f72e8acd2c 100644 --- a/erts/doc/src/alt_dist.xml +++ b/erts/doc/src/alt_dist.xml @@ -680,7 +680,10 @@ of the distribution controller, <c>Received</c> is received packets, <c>Sent</c> is sent packets, and <c>PendSend</c> is - amount of packets in queue to be sent + amount of data in queue to be sent + (typically in bytes, but <c>dist_util</c> + only checks whether the value is non-zero + to know there is data in queue) or a <c>boolean()</c> indicating whether there are packets in queue to be sent. </p> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index ef55b2ed0a..cfa952f01c 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,34 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 10.4.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a buffer overflow when + <c>binary_to_existing_atom/2</c> and + <c>list_to_existing_atom/2</c> was used with the + <c>latin1</c> encoding.</p> + <p> + Own Id: OTP-15819 Aux Id: ERL-944 </p> + </item> + <item> + <p> + The runtime system disconnected a connection if it + received an <c>exit/2</c> signal where the recipient was + a process on an old incarnation of the current node. That + is, the receiving node had the same node name, but + another "creation" number. The signal will now just be + dropped since the receiving process no longer exists.</p> + <p> + Own Id: OTP-15867 Aux Id: ERIERL-373 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.4.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index db9c258cb7..602db106b1 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -413,6 +413,7 @@ bif re:compile/1 bif re:compile/2 bif re:run/2 bif re:run/3 +bif re:internal_run/4 # # Bifs in lists module. diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 52fcfe3545..4537e3e569 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -3342,7 +3342,7 @@ dist_get_stat_1(BIF_ALIST_1) am_ok, erts_bld_sint64(hpp, szp, read), erts_bld_sint64(hpp, szp, write), - pend ? am_true : am_false); + erts_bld_sint64(hpp, szp, pend)); if (hpp) break; hp = HAlloc(BIF_P, sz); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index e0b9202fe7..b3bf1c7ee3 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -46,7 +46,7 @@ static Export *urun_trap_exportp = NULL; static Export *ucompile_trap_exportp = NULL; static BIF_RETTYPE re_exec_trap(BIF_ALIST_3); -static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first); static void *erts_erts_pcre_malloc(size_t size) { return erts_alloc(ERTS_ALC_T_RE_HEAP,size); @@ -1094,7 +1094,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) * The actual re:run/2,3 BIFs */ static BIF_RETTYPE -re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first) { const pcre *code_tmp; RestartContext restart; @@ -1120,6 +1120,14 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) < 0) { BIF_ERROR(p,BADARG); } + if (!first) { + /* + * 'first' is false when re:grun() previously has called re:internal_run() + * with the same subject; i.e., no need to do yet another validation of + * the subject regarding utf8 encoding... + */ + options |= PCRE_NO_UTF8_CHECK; + } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); @@ -1360,15 +1368,28 @@ handle_iolist: } BIF_RETTYPE +re_internal_run_4(BIF_ALIST_4) +{ + int first; + if (BIF_ARG_4 == am_false) + first = 0; + else if (BIF_ARG_4 == am_true) + first = !0; + else + BIF_ERROR(BIF_P,BADARG); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, first); +} + +BIF_RETTYPE re_run_3(BIF_ALIST_3) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, !0); } BIF_RETTYPE re_run_2(BIF_ALIST_2) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL, !0); } /* @@ -1407,6 +1428,7 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3) loop_count = 0xFFFFFFFF; #endif rc = erts_pcre_exec(NULL, &(restartp->extra), NULL, 0, 0, 0, NULL, 0); + ASSERT(loop_count != 0xFFFFFFFF); BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { diff --git a/erts/emulator/beam/erl_db_catree.c b/erts/emulator/beam/erl_db_catree.c index e0d5e44f58..fed4b44a9b 100644 --- a/erts/emulator/beam/erl_db_catree.c +++ b/erts/emulator/beam/erl_db_catree.c @@ -166,8 +166,17 @@ static void split_catree(DbTableCATree *tb, static void join_catree(DbTableCATree *tb, DbTableCATreeNode *thiz, DbTableCATreeNode *parent); - - +static ERTS_INLINE +int try_wlock_base_node(DbTableCATreeBaseNode *base_node); +static ERTS_INLINE +void wunlock_base_node(DbTableCATreeNode *base_node); +static ERTS_INLINE +void wlock_base_node_no_stats(DbTableCATreeNode *base_node); +static ERTS_INLINE +void wunlock_adapt_base_node(DbTableCATree* tb, + DbTableCATreeNode* node, + DbTableCATreeNode* parent, + int current_level); /* ** External interface */ @@ -210,12 +219,16 @@ DbTableMethod db_catree = * Constants */ -#define ERL_DB_CATREE_LOCK_FAILURE_CONTRIBUTION 200 +#define ERL_DB_CATREE_LOCK_FAILURE_CONTRIBUTION 250 #define ERL_DB_CATREE_LOCK_SUCCESS_CONTRIBUTION (-1) +#define ERL_DB_CATREE_LOCK_GRAVITY_CONTRIBUTION (-500) +#define ERL_DB_CATREE_LOCK_GRAVITY_PATTERN (0xFF800000) #define ERL_DB_CATREE_LOCK_MORE_THAN_ONE_CONTRIBUTION (-10) #define ERL_DB_CATREE_HIGH_CONTENTION_LIMIT 1000 #define ERL_DB_CATREE_LOW_CONTENTION_LIMIT (-1000) -#define ERL_DB_CATREE_MAX_ROUTE_NODE_LAYER_HEIGHT 14 +#define ERL_DB_CATREE_MAX_ROUTE_NODE_LAYER_HEIGHT 16 +#define ERL_DB_CATREE_LOCK_LOW_NO_CONTRIBUTION_LIMIT (-20000) +#define ERL_DB_CATREE_LOCK_HIGH_NO_CONTRIBUTION_LIMIT (20000) /* * Internal CA tree related helper functions and macros @@ -245,6 +258,27 @@ DbTableMethod db_catree = #define SET_LEFT_RELB(ca_tree_route_node, v) erts_atomic_set_relb(&(ca_tree_route_node->u.route.left), (erts_aint_t)(v)); #define SET_RIGHT_RELB(ca_tree_route_node, v) erts_atomic_set_relb(&(ca_tree_route_node->u.route.right), (erts_aint_t)(v)); +/* Change base node lock statistics */ +#define BASE_NODE_STAT_SET(NODE, VALUE) erts_atomic_set_nob(&(NODE)->u.base.lock_statistics, VALUE) +#define BASE_NODE_STAT_READ(NODE) erts_atomic_read_nob(&(NODE)->u.base.lock_statistics) +#define BASE_NODE_STAT_ADD(NODE, VALUE) \ + do { \ + Sint v = erts_atomic_read_nob(&((NODE)->u.base.lock_statistics)); \ + ASSERT(VALUE > 0); \ + if(v < ERL_DB_CATREE_LOCK_HIGH_NO_CONTRIBUTION_LIMIT) { \ + erts_atomic_set_nob(&(NODE->u.base.lock_statistics), v + VALUE); \ + } \ + }while(0); +#define BASE_NODE_STAT_SUB(NODE, VALUE) \ + do { \ + Sint v = erts_atomic_read_nob(&((NODE)->u.base.lock_statistics)); \ + ASSERT(VALUE < 0); \ + if(v > ERL_DB_CATREE_LOCK_LOW_NO_CONTRIBUTION_LIMIT) { \ + erts_atomic_set_nob(&(NODE->u.base.lock_statistics), v + VALUE); \ + } \ + }while(0); + + /* Compares a key to the key in a route node */ static ERTS_INLINE Sint cmp_key_route(Eterm key, DbTableCATreeNode *obj) @@ -653,10 +687,10 @@ static void dbg_provoke_random_splitjoin(DbTableCATree* tb, switch (dbg_fastrand() % 8) { case 1: - base_node->u.base.lock_statistics = 1+ERL_DB_CATREE_HIGH_CONTENTION_LIMIT; + BASE_NODE_STAT_ADD(base_node, 1+ERL_DB_CATREE_HIGH_CONTENTION_LIMIT); break; case 2: - base_node->u.base.lock_statistics = -1+ERL_DB_CATREE_LOW_CONTENTION_LIMIT; + BASE_NODE_STAT_SUB(base_node, -1+ERL_DB_CATREE_LOW_CONTENTION_LIMIT); break; } } @@ -664,6 +698,48 @@ static void dbg_provoke_random_splitjoin(DbTableCATree* tb, # define dbg_provoke_random_splitjoin(T,N) #endif /* PROVOKE_RANDOM_SPLIT_JOIN */ +static ERTS_NOINLINE +void do_random_join(DbTableCATree* tb, Uint rand) +{ + DbTableCATreeNode* node = GET_ROOT_ACQB(tb); + DbTableCATreeNode* parent = NULL; + int level = 0; + Sint stat; + while (!node->is_base_node) { + parent = node; + if ((rand & (1 << level)) == 0) { + node = GET_LEFT_ACQB(node); + } else { + node = GET_RIGHT_ACQB(node); + } + level++; + } + BASE_NODE_STAT_SUB(node, ERL_DB_CATREE_LOCK_GRAVITY_CONTRIBUTION); + stat = BASE_NODE_STAT_READ(node); + if (stat >= ERL_DB_CATREE_LOW_CONTENTION_LIMIT && + stat <= ERL_DB_CATREE_HIGH_CONTENTION_LIMIT) { + return; /* No adaptation */ + } + if (parent != NULL && !try_wlock_base_node(&node->u.base)) { + if (!node->u.base.is_valid) { + wunlock_base_node(node); + return; + } + wunlock_adapt_base_node(tb, node, parent, level); + } +} + +static ERTS_INLINE +void do_random_join_with_low_probability(DbTableCATree* tb, Uint seed) +{ +#ifndef ERTS_DB_CA_TREE_NO_RANDOM_JOIN_WITH_LOW_PROBABILITY + Uint32 rand = erts_sched_local_random(seed); + if (((rand & ERL_DB_CATREE_LOCK_GRAVITY_PATTERN)) == 0) { + do_random_join(tb, rand); + } +#endif +} + static ERTS_INLINE int try_wlock_base_node(DbTableCATreeBaseNode *base_node) { @@ -691,9 +767,9 @@ void wlock_base_node(DbTableCATreeNode *base_node) if (try_wlock_base_node(&base_node->u.base)) { /* The lock is contended */ wlock_base_node_no_stats(base_node); - base_node->u.base.lock_statistics += ERL_DB_CATREE_LOCK_FAILURE_CONTRIBUTION; + BASE_NODE_STAT_ADD(base_node, ERL_DB_CATREE_LOCK_FAILURE_CONTRIBUTION); } else { - base_node->u.base.lock_statistics += ERL_DB_CATREE_LOCK_SUCCESS_CONTRIBUTION; + BASE_NODE_STAT_SUB(base_node, ERL_DB_CATREE_LOCK_SUCCESS_CONTRIBUTION); } } @@ -709,13 +785,14 @@ void wunlock_adapt_base_node(DbTableCATree* tb, DbTableCATreeNode* parent, int current_level) { + Sint base_node_lock_stat = BASE_NODE_STAT_READ(node); dbg_provoke_random_splitjoin(tb,node); if ((!node->u.base.root && parent && !(tb->common.status & DB_CATREE_FORCE_SPLIT)) - || node->u.base.lock_statistics < ERL_DB_CATREE_LOW_CONTENTION_LIMIT) { + || base_node_lock_stat < ERL_DB_CATREE_LOW_CONTENTION_LIMIT) { join_catree(tb, node, parent); } - else if (node->u.base.lock_statistics > ERL_DB_CATREE_HIGH_CONTENTION_LIMIT + else if (base_node_lock_stat > ERL_DB_CATREE_HIGH_CONTENTION_LIMIT && current_level < ERL_DB_CATREE_MAX_ROUTE_NODE_LAYER_HEIGHT) { split_catree(tb, node, parent); } @@ -728,11 +805,23 @@ static ERTS_INLINE void rlock_base_node(DbTableCATreeNode *base_node) { ASSERT(base_node->is_base_node); - erts_rwmtx_rlock(&base_node->u.base.lock); + if (EBUSY == erts_rwmtx_tryrlock(&base_node->u.base.lock)) { + /* The lock is contended */ + BASE_NODE_STAT_ADD(base_node, ERL_DB_CATREE_LOCK_FAILURE_CONTRIBUTION); + erts_rwmtx_rlock(&base_node->u.base.lock); + } +} + +static ERTS_INLINE +void runlock_base_node(DbTableCATreeNode *base_node, DbTableCATree* tb) +{ + ASSERT(base_node->is_base_node); + erts_rwmtx_runlock(&base_node->u.base.lock); + do_random_join_with_low_probability(tb, (Uint)base_node); } static ERTS_INLINE -void runlock_base_node(DbTableCATreeNode *base_node) +void runlock_base_node_no_rand(DbTableCATreeNode *base_node) { ASSERT(base_node->is_base_node); erts_rwmtx_runlock(&base_node->u.base.lock); @@ -814,7 +903,7 @@ void unlock_iter_base_node(CATreeRootIterator* iter) { ASSERT(iter->locked_bnode); if (iter->read_only) - runlock_base_node(iter->locked_bnode); + runlock_base_node(iter->locked_bnode, iter->tb); else if (iter->locked_bnode->u.base.is_valid) { wunlock_adapt_base_node(iter->tb, iter->locked_bnode, iter->bnode_parent, iter->bnode_level); @@ -874,7 +963,7 @@ DbTableCATreeNode* find_rlock_valid_base_node(DbTableCATree* tb, Eterm key) rlock_base_node(base_node); if (base_node->u.base.is_valid) break; - runlock_base_node(base_node); + runlock_base_node_no_rand(base_node); } return base_node; } @@ -923,8 +1012,8 @@ static DbTableCATreeNode *create_base_node(DbTableCATree *tb, "erl_db_catree_base_node", NIL, ERTS_LOCK_FLAGS_CATEGORY_DB); - p->u.base.lock_statistics = ((tb->common.status & DB_CATREE_FORCE_SPLIT) - ? INT_MAX : 0); + BASE_NODE_STAT_SET(p, ((tb->common.status & DB_CATREE_FORCE_SPLIT) + ? INT_MAX : 0)); p->u.base.is_valid = 1; return p; } @@ -1094,7 +1183,7 @@ static void join_catree(DbTableCATree *tb, ASSERT(thiz->is_base_node); if (parent == NULL) { - thiz->u.base.lock_statistics = 0; + BASE_NODE_STAT_SET(thiz, 0); wunlock_base_node(thiz); return; } @@ -1103,11 +1192,11 @@ static void join_catree(DbTableCATree *tb, neighbor = leftmost_base_node(GET_RIGHT_ACQB(parent)); if (try_wlock_base_node(&neighbor->u.base)) { /* Failed to acquire lock */ - thiz->u.base.lock_statistics = 0; + BASE_NODE_STAT_SET(thiz, 0); wunlock_base_node(thiz); return; } else if (!neighbor->u.base.is_valid) { - thiz->u.base.lock_statistics = 0; + BASE_NODE_STAT_SET(thiz, 0); wunlock_base_node(thiz); wunlock_base_node(neighbor); return; @@ -1153,11 +1242,11 @@ static void join_catree(DbTableCATree *tb, neighbor = rightmost_base_node(GET_LEFT_ACQB(parent)); if (try_wlock_base_node(&neighbor->u.base)) { /* Failed to acquire lock */ - thiz->u.base.lock_statistics = 0; + BASE_NODE_STAT_SET(thiz, 0); wunlock_base_node(thiz); return; } else if (!neighbor->u.base.is_valid) { - thiz->u.base.lock_statistics = 0; + BASE_NODE_STAT_SET(thiz, 0); wunlock_base_node(thiz); wunlock_base_node(neighbor); return; @@ -1241,7 +1330,7 @@ static void split_catree(DbTableCATree *tb, if (less_than_two_elements(base->u.base.root)) { if (!(tb->common.status & DB_CATREE_FORCE_SPLIT)) - base->u.base.lock_statistics = 0; + BASE_NODE_STAT_SET(base, 0); wunlock_base_node(base); return; } else { @@ -1521,7 +1610,7 @@ static int db_get_catree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) int result = db_get_tree_common(p, &tb->common, node->u.base.root, key, ret, NULL); - runlock_base_node(node); + runlock_base_node(node, tb); return result; } @@ -1804,7 +1893,7 @@ static int db_member_catree(DbTable *tbl, Eterm key, Eterm *ret) int result = db_member_tree_common(&tb->common, node->u.base.root, key, ret, NULL); - runlock_base_node(node); + runlock_base_node(node, tb); return result; } @@ -1816,7 +1905,7 @@ static int db_get_element_catree(Process *p, DbTable *tbl, int result = db_get_element_tree_common(p, &tb->common, node->u.base.root, key, ndex, ret, NULL); - runlock_base_node(node); + runlock_base_node(node, tb); return result; } @@ -2250,7 +2339,7 @@ void db_catree_force_split(DbTableCATree* tb, int on) init_root_iterator(tb, &iter, 1); root = catree_find_first_root(&iter); do { - iter.locked_bnode->u.base.lock_statistics = (on ? INT_MAX : 0); + BASE_NODE_STAT_SET(iter.locked_bnode, (on ? INT_MAX : 0)); root = catree_find_next_root(&iter, NULL); } while (root); destroy_root_iterator(&iter); diff --git a/erts/emulator/beam/erl_db_catree.h b/erts/emulator/beam/erl_db_catree.h index cf3498dabb..c2c884eee3 100644 --- a/erts/emulator/beam/erl_db_catree.h +++ b/erts/emulator/beam/erl_db_catree.h @@ -42,7 +42,7 @@ typedef struct { typedef struct { erts_rwmtx_t lock; /* The lock for this base node */ - Sint lock_statistics; + erts_atomic_t lock_statistics; int is_valid; /* If this base node is still valid */ TreeDbTerm *root; /* The root of the sequential tree */ ErtsThrPrgrLaterOp free_item; /* Used when freeing using thread progress */ diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 72534b2bf0..fb900ca7ba 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -4051,6 +4051,7 @@ erts_proc_sig_signal_size(ErtsSignal *sig) case ERTS_MON_TYPE_DIST_PROC: case ERTS_MON_TYPE_NODE: size = erts_monitor_size((ErtsMonitor *) sig); + break; default: ERTS_INTERNAL_ERROR("Unexpected sig type"); break; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index bbf50b4189..0d6b512f78 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -2632,6 +2632,9 @@ void erts_notify_inc_runq(ErtsRunQueue *runq); void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); void erts_aux_thread_poke(void); +ERTS_GLB_INLINE Uint32 erts_sched_local_random_hash_64_to_32_shift(Uint64 key); +ERTS_GLB_INLINE Uint32 erts_sched_local_random(Uint additional_seed); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -2648,6 +2651,39 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) } +/* + * Source: https://gist.github.com/badboy/6267743 + * http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm + */ +ERTS_GLB_INLINE +Uint32 erts_sched_local_random_hash_64_to_32_shift(Uint64 key) +{ + key = (~key) + (key << 18); /* key = (key << 18) - key - 1; */ + key = key ^ (key >> 31); + key = (key + (key << 2)) + (key << 4); + key = key ^ (key >> 11); + key = key + (key << 6); + key = key ^ (key >> 22); + return (Uint32) key; +} + +/* + * This function attempts to return a random number based on the state + * of the scheduler, the current process and the additional_seed + * parameter. + */ +ERTS_GLB_INLINE +Uint32 erts_sched_local_random(Uint additional_seed) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint64 seed = + additional_seed + + esdp->reductions + + esdp->current_process->fcalls + + (((Uint64)esdp->no) << 32); + return erts_sched_local_random_hash_64_to_32_shift(seed); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9d68a4a68b..b9d4f6afcc 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1692,6 +1692,7 @@ i_increment rxy W d # Handle unoptimized code. i_plus S1=c S2=c Fail Dst => move S1 x | i_plus x S2 Fail Dst +i_plus S1=c S2=xy Fail Dst => i_plus S2 S1 Fail Dst i_plus xy xyc j? d diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index c5e2af0b5e..3e1d7e4d5e 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -18,6 +18,7 @@ * %CopyrightEnd% */ +#ifdef __arm__ #include <stddef.h> /* offsetof() */ #ifdef HAVE_CONFIG_H @@ -285,3 +286,5 @@ void hipe_arch_print_pcb(struct hipe_process_state *p) U("narity ", narity); #undef U } + +#endif /*__arm__*/ diff --git a/erts/emulator/hipe/hipe_x86.h b/erts/emulator/hipe/hipe_x86.h index 8967793171..a1fe75e792 100644 --- a/erts/emulator/hipe/hipe_x86.h +++ b/erts/emulator/hipe/hipe_x86.h @@ -22,17 +22,28 @@ #ifndef HIPE_X86_H #define HIPE_X86_H -static __inline__ void hipe_flush_icache_word(void *address) -{ - /* Do nothing. This works as long as compiled code is - executed by a single CPU thread. */ -} +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif static __inline__ void hipe_flush_icache_range(void *address, unsigned int nbytes) { - /* Do nothing. This works as long as compiled code is - executed by a single CPU thread. */ + void* end = (char*)address + nbytes; + +#if ERTS_AT_LEAST_GCC_VSN__(4, 3, 0) || __has_builtin(__builtin___clear_cache) + __builtin___clear_cache(address, end); +#elif defined(__clang__) + void __clear_cache(void *start, void *end); + __clear_cache(address, end); +#else +# warning "Don't know how to flush instruction cache" +#endif +} + +static __inline__ void hipe_flush_icache_word(void *address) +{ + hipe_flush_icache_range(address, sizeof(void*)); } /* for stack descriptor hash lookup */ diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h index 3563791223..505e2ccce0 100644 --- a/erts/emulator/pcre/pcre.h +++ b/erts/emulator/pcre/pcre.h @@ -240,6 +240,9 @@ with J. */ #define PCRE_UTF8_ERR20 20 #define PCRE_UTF8_ERR21 21 #define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ +#if defined(ERLANG_INTEGRATION) +#define PCRE_UTF8_YIELD 23 +#endif /* Specific error codes for UTF-16 validity checks */ diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 1946e97a72..55a7b377bf 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -6642,10 +6642,16 @@ typedef struct { REAL_PCRE *Xre; heapframe Xframe_zero; /* Always NO_RECURSE */ + /* for yield in valid_utf() */ + + struct PRIV(valid_utf_ystate) valid_utf_ystate; + /* Original function parameters that need be saved */ int Xstart_offset; int Xoffsetcount; int *Xoffsets; + int Xlength; + PCRE_SPTR Xsubject; } PcreExecContext; #endif @@ -6675,6 +6681,7 @@ pcre32_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, #endif { #ifndef ERLANG_INTEGRATION +#define ERTS_UPDATE_CONSUMED(X, MD) int rc, ocount, arg_offset_max; int newline; BOOL using_temporary_offsets = FALSE; @@ -6736,6 +6743,8 @@ heapframe frame_zero; start_offset = exec_context->Xstart_offset; \ offsetcount = exec_context->Xoffsetcount; \ offsets = exec_context->Xoffsets; \ + length = exec_context->Xlength; \ + subject = exec_context->Xsubject; \ } while (0) #define SWAPOUT() do { \ @@ -6750,8 +6759,30 @@ heapframe frame_zero; exec_context->Xstart_offset = start_offset; \ exec_context->Xoffsetcount = offsetcount; \ exec_context->Xoffsets = offsets; \ + exec_context->Xlength = length; \ + exec_context->Xsubject = subject; \ } while (0) +#define ERTS_UPDATE_CONSUMED(X, MD) \ +do { \ + if (((X)->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { \ + unsigned long consumed__; \ + if (!(X)->restart_data) { \ + consumed__ = 0; \ + } \ + else { \ + PcreExecContext *ctx__ = (PcreExecContext *) \ + (*(X)->restart_data); \ + consumed__ = ctx__->valid_utf_ystate.cnt; \ + ctx__->valid_utf_ystate.cnt = 0; \ + } \ + if ((MD)) { \ + match_data *md__ = (MD); \ + consumed__ += (X)->loop_limit - md__->loop_limit; \ + } \ + *((X)->loop_counter_return) = consumed__; \ + } \ +} while (0) PcreExecContext *exec_context; PcreExecContext internal_context; @@ -6776,15 +6807,21 @@ pcre_uchar req_char; /* we are restarting, every initialization is skipped and we jump directly into the loop */ exec_context = (PcreExecContext *) *(extra_data->restart_data); SWAPIN(); - + if (exec_context->valid_utf_ystate.yielded) + goto restart_valid_utf; goto RESTART_INTERRUPTED; } else { if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { exec_context = (PcreExecContext *) (erts_pcre_malloc)(sizeof(PcreExecContext)); - *(extra_data->restart_data) = (void *) exec_context; + *(extra_data->restart_data) = (void *) exec_context; + exec_context->valid_utf_ystate.yielded = 0; /* need freeing by special routine from client */ } else { +#if defined(ERLANG_INTEGRATION) + fprintf(stderr, "Unexpected execution path\n"); + abort(); +#endif exec_context = &internal_context; } @@ -6865,9 +6902,38 @@ code for an invalid string if a results vector is available. */ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) { int erroroffset; - int errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length, &erroroffset); + int errorcode; + +#if !defined(ERLANG_INTEGRATION) + errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length); +#else + struct PRIV(valid_utf_ystate) *ystate; + + if (!extra_data || !extra_data->restart_data) { + ystate = NULL; + } + else if (!(extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { + exec_context->valid_utf_ystate.cnt = 10; + ystate = NULL; + } + else { + exec_context->valid_utf_ystate.yielded = 0; + restart_valid_utf: + ystate = &exec_context->valid_utf_ystate; + ystate->cnt = (int) extra_data->loop_limit; + } + errorcode = PRIV(yielding_valid_utf)((PCRE_PUCHAR)subject, length, + &erroroffset, ystate); +#endif if (errorcode != 0) { +#if defined(ERLANG_INTEGRATION) + if (ystate && ystate->yielded) { + ERTS_UPDATE_CONSUMED(extra_data, NULL); + SWAPOUT(); + return PCRE_ERROR_LOOP_LIMIT; + } +#endif if (offsetcount >= 2) { offsets[0] = erroroffset; @@ -6890,6 +6956,11 @@ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) return PCRE_ERROR_BADUTF8_OFFSET; #endif } +#if defined(ERLANG_INTEGRATION) +else { + exec_context->valid_utf_ystate.cnt = 0; +} +#endif #endif /* If the pattern was successfully studied with JIT support, run the JIT @@ -6950,7 +7021,11 @@ if (extra_data != NULL) #ifdef ERLANG_INTEGRATION if ((flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { - md->loop_limit = extra_data->loop_limit; + md->loop_limit = extra_data->loop_limit; + if (extra_data->restart_data) + md->loop_limit -= extra_data->loop_limit - exec_context->valid_utf_ystate.cnt; + if (md->loop_limit < 10) + md->loop_limit = 10; /* At least do something if we've come this far... */ } #endif } @@ -7266,14 +7341,8 @@ for(;;) #endif if ((start_bits[c/8] & (1 << (c&7))) != 0) { -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif - break; + ERTS_UPDATE_CONSUMED(extra_data, md); + break; } start_match++; } @@ -7298,13 +7367,7 @@ for(;;) (pcre_uint32)(end_subject - start_match) < study->minlength) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7353,13 +7416,7 @@ for(;;) if (p >= end_subject) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7390,11 +7447,7 @@ for(;;) EDEBUGF(("Calling match...")); rc = match(start_match, md->start_code, start_match, 2, md, NULL, 0); #ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } + ERTS_UPDATE_CONSUMED(extra_data, md); SWAPOUT(); while(rc == PCRE_ERROR_LOOP_LIMIT) { EDEBUGF(("Loop limit break detected")); diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h index c84dcb5a38..71f473e86f 100644 --- a/erts/emulator/pcre/pcre_internal.h +++ b/erts/emulator/pcre/pcre_internal.h @@ -2756,6 +2756,17 @@ extern int PRIV(strcmp_uc_c8_utf)(const pcre_uchar *, #endif /* COMPILE_PCRE[8|16|32] */ +#if defined(ERLANG_INTEGRATION) +struct PRIV(valid_utf_ystate) { + unsigned int cnt; + int length; + int yielded; + PCRE_PUCHAR p; +}; +extern int PRIV(yielding_valid_utf)(PCRE_PUCHAR, int, int *, + struct PRIV(valid_utf_ystate) *); +#endif + extern const pcre_uchar *PRIV(find_bracket)(const pcre_uchar *, BOOL, int); extern BOOL PRIV(is_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, int *, BOOL); diff --git a/erts/emulator/pcre/pcre_valid_utf8.c b/erts/emulator/pcre/pcre_valid_utf8.c index 516d8f4725..1dc1f9ba0c 100644 --- a/erts/emulator/pcre/pcre_valid_utf8.c +++ b/erts/emulator/pcre/pcre_valid_utf8.c @@ -107,19 +107,80 @@ Returns: = 0 if the string is a valid UTF-8 string int PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) { + +#if defined(ERLANG_INTEGRATION) + return PRIV(yielding_valid_utf)(string, length, erroroffset, NULL); +} + +int +PRIV(yielding_valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset, struct PRIV(valid_utf_ystate) *ystate) +{ +#endif + #ifdef SUPPORT_UTF register PCRE_PUCHAR p; +#if defined(ERLANG_INTEGRATION) +register long cnt; + +if (!ystate) { + cnt = -1; +} +else { + cnt = ystate->cnt; + if (ystate->yielded) { + p = ystate->p; + length = ystate->length; + if (length < 0) + goto restart_length; + else + goto restart_validate; + } +} +#endif + if (length < 0) { - for (p = string; *p != 0; p++); - length = (int)(p - string); + for (p = string; *p != 0; p++) { +#if defined(ERLANG_INTEGRATION) + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + restart_length: + (void) !0; +#endif + } + length = (int)(p - string); } for (p = string; length-- > 0; p++) { register pcre_uchar ab, c, d; +#if defined(ERLANG_INTEGRATION) + + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + + restart_validate: + +#endif + c = *p; if (c < 128) continue; /* ASCII character */ @@ -290,6 +351,14 @@ for (p = string; length-- > 0; p++) } } +#if defined(ERLANG_INTEGRATION) +if (ystate) { + /* Return with cnt set to amount consumed... */ + ystate->cnt -= cnt; + ystate->yielded = 0; +} +#endif + #else /* Not SUPPORT_UTF */ (void)(string); /* Keep picky compilers happy */ (void)(length); diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 55b1162cfb..b48be3dd04 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -37,10 +37,13 @@ -export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1, + procs_bug/1, ets_count/1, atom_count/1, system_logger/1]). -export([init/1, handle_event/2, handle_call/2]). +-export([init_per_testcase/2, end_per_testcase/2]). + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 2}}]. @@ -48,8 +51,20 @@ suite() -> all() -> [process_count, system_version, misc_smoke_tests, ets_count, heap_size, wordsize, memory, ets_limit, atom_limit, atom_count, + procs_bug, system_logger]. + +init_per_testcase(procs_bug, Config) -> + procs_bug(init_per_testcase, Config); +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(procs_bug, Config) -> + procs_bug(end_per_testcase, Config); +end_per_testcase(_, _) -> + ok. + %%% %%% The test cases ------------------------------------------------------------- %%% @@ -654,3 +669,41 @@ handle_call(Msg, State) -> handle_event(Event, State) -> State ! {report_handler, Event}, {ok, State}. + + +%% OTP-15909: Provoke bug that would cause VM crash +%% if doing system_info(procs) when process have queued exit/down signals. +procs_bug(init_per_testcase, Config) -> + %% Use single scheduler and process prio to starve monitoring processes + %% from handling their received DOWN signals. + OldSchedOnline = erlang:system_flag(schedulers_online,1), + [{schedulers_online, OldSchedOnline} | Config]; +procs_bug(end_per_testcase, Config) -> + erlang:system_flag(schedulers_online, + proplists:get_value(schedulers_online, Config)), + ok. + +procs_bug(Config) when is_list(Config) -> + {Monee,_} = spawn_opt(fun () -> receive die -> ok end end, + [monitor,{priority,max}]), + Papa = self(), + Pids = [begin + P = spawn_opt(fun () -> + erlang:monitor(process, Monee), + Papa ! {self(),ready}, + receive "nada" -> no end + end, + [link, {priority,normal}]), + {P, ready} = receive M -> M end, + P + end + || _ <- lists:seq(1,10)], + process_flag(priority,high), + Monee ! die, + {'DOWN',_,process,Monee,normal} = receive M -> M end, + + %% This call did crash VM as Pids have pending DOWN signals. + erlang:system_info(procs), + process_flag(priority,normal), + [begin unlink(P), exit(P, kill) end || P <- Pids], + ok. diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 8203c46a39..c793243c13 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -255,7 +255,9 @@ static char* key_val_name = ERLANG_VERSION; /* Used by the registry * access functions. */ static char* boot_script = NULL; /* used by option -start_erl and -boot */ -static char* config_script = NULL; /* used by option -start_erl and -config */ +static char** config_scripts = NULL; /* used by option -start_erl and -config */ +static int config_script_cnt = 0; +static int got_start_erl = 0; static HANDLE this_module_handle; static int run_werl; @@ -392,6 +394,22 @@ add_extra_suffixes(char *prog) } #ifdef __WIN32__ +static void add_boot_config(void) +{ + int i; + if (boot_script) + add_args("-boot", boot_script, NULL); + for (i = 0; i < config_script_cnt; i++) { + add_args("-config", config_scripts[i], NULL); + } +} +# define ADD_BOOT_CONFIG add_boot_config() +#else +# define ADD_BOOT_CONFIG +#endif + + +#ifdef __WIN32__ __declspec(dllexport) int win_erlexec(int argc, char **argv, HANDLE module, int windowed) #else int main(int argc, char **argv) @@ -581,16 +599,6 @@ int main(int argc, char **argv) i = 1; -#ifdef __WIN32__ -#define ADD_BOOT_CONFIG \ - if (boot_script) \ - add_args("-boot", boot_script, NULL); \ - if (config_script) \ - add_args("-config", config_script, NULL); -#else -#define ADD_BOOT_CONFIG -#endif - get_home(); add_args("-home", home, NULL); @@ -610,7 +618,9 @@ int main(int argc, char **argv) case 'b': if (strcmp(argv[i], "-boot") == 0) { if (boot_script) - error("Conflicting -start_erl and -boot options"); + error("Conflicting -boot options"); + if (got_start_erl) + error("Conflicting -start_erl and -boot options"); if (i+1 >= argc) usage("-boot"); boot_script = strsave(argv[i+1]); @@ -634,11 +644,14 @@ int main(int argc, char **argv) } #ifdef __WIN32__ else if (strcmp(argv[i], "-config") == 0){ - if (config_script) + if (got_start_erl) error("Conflicting -start_erl and -config options"); if (i+1 >= argc) usage("-config"); - config_script = strsave(argv[i+1]); + config_script_cnt++; + config_scripts = erealloc(config_scripts, + config_script_cnt * sizeof(char*)); + config_scripts[config_script_cnt-1] = strsave(argv[i+1]); i++; } #endif @@ -1371,6 +1384,7 @@ strsave(char* string) static void get_start_erl_data(char *file) { + static char* a_config_script; int fp; char tmpbuffer[512]; char start_erl_data[512]; @@ -1381,7 +1395,7 @@ static void get_start_erl_data(char *file) char* tprogname; if (boot_script) error("Conflicting -start_erl and -boot options"); - if (config_script) + if (config_scripts) error("Conflicting -start_erl and -config options"); env = get_env("RELDIR"); if (env) @@ -1431,10 +1445,13 @@ static void get_start_erl_data(char *file) erts_snprintf(progname,strlen(tprogname) + 20,"%s -start_erl",tprogname); boot_script = emalloc(512); - config_script = emalloc(512); + a_config_script = emalloc(512); erts_snprintf(boot_script, 512, "%s/%s/start", reldir, otpstring); - erts_snprintf(config_script, 512, "%s/%s/sys", reldir, otpstring); + erts_snprintf(a_config_script, 512, "%s/%s/sys", reldir, otpstring); + config_scripts = &a_config_script; + config_script_cnt = 1; + got_start_erl = 1; } diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 8cfc2d549e..710a7a9ef6 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -292,10 +292,20 @@ if [ "x$GDB" = "x" ]; then valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log" fi fi - if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then - valgrind_misc_flags= - else - valgrind_misc_flags="$VALGRIND_MISC_FLAGS" + # Add default flags + vgflags=$VALGRIND_MISC_FLAGS + if [ "x${vgflags#*--show-possibly-lost}" = "x$vgflags" ]; then + vgflags="$vgflags --show-possibly-lost=no" + fi + if [ "x${vgflags#*--child-silent-after-fork}" = "x$vgflags" ]; then + vgflags="$vgflags --child-silent-after-fork=yes" + fi + if [ "x${vgflags#*--suppressions}" = "x$vgflags" ]; then + if [ "x$ERL_TOP" != "x" ]; then + vgflags="$vgflags --suppressions=$ERL_TOP/erts/emulator/valgrind/suppress.standard" + else + echo "No valgrind suppression file found in \$VALGRIND_MISC_FLAGS and \$ERL_TOP not set." + fi fi if which taskset > /dev/null && test -e /proc/cpuinfo; then # We only let valgrind utilize one core with "taskset 1" as it can be very slow @@ -310,7 +320,7 @@ if [ "x$GDB" = "x" ]; then sched_arg= fi - exec $taskset1 valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" + exec $taskset1 valgrind $valgrind_xml $valgrind_log $vgflags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" elif [ $run_rr = yes ]; then if [ $1 = replay ]; then diff --git a/erts/etc/unix/gcov-gen-html b/erts/etc/unix/gcov-gen-html new file mode 100755 index 0000000000..3fd9f1ca49 --- /dev/null +++ b/erts/etc/unix/gcov-gen-html @@ -0,0 +1,62 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +@ARGV == 2 or die "Usage: gcov-gen-html \$ERL_TOP <output directory>\n"; + +my $srcdir = shift @ARGV; +my $outdir = shift @ARGV; + +my $verbose = 1; +my $flavor = "smp"; + +# setup filenames and paths, observe geninfos --base-directory +# it needs a correct path just after the $geninfo + +my $lcov_path = ""; #/usr/local/bin/"; + +my $geninfo = $lcov_path . "geninfo --no-checksum --base-directory"; +my $genhtml = $lcov_path . "genhtml"; + +# src paths + +my $emu_src_path = "$srcdir/erts/emulator"; +my $elib_src_path = "$srcdir/erts/lib_src"; +my $pcre_src_path = "$emu_src_path"; +my $zlib_src_path = "$emu_src_path"; + +# obj paths + +my $emu_obj_path = <$emu_src_path/obj/*-linux-*/gcov/$flavor>; +my $elib_obj_path = <$elib_src_path/obj/*-linux-*/gcov>; +my $pcre_obj_path = <$emu_src_path/pcre/obj/*-linux-*/gcov>; +my $zlib_obj_path = <$emu_src_path/zlib/obj/*-linux-*/gcov>; + +# info files + +my $emu_info = "$srcdir/emulator-cover.info"; +my $elib_info = "$srcdir/elib-cover.info"; +my $pcre_info = "$srcdir/pcre-cover.info"; + +my $infofiles = "$emu_info $elib_info $pcre_info"; + +run("$geninfo $emu_src_path -o $emu_info $emu_obj_path"); +run("$geninfo $elib_src_path -o $elib_info $elib_obj_path"); +run("$geninfo $pcre_src_path -o $pcre_info $pcre_obj_path"); + +if (<$zlib_obj_path/*.o>) { + my $zlib_info = "$srcdir/zlib-cover.info"; + $infofiles .= " $zlib_info"; + run("$geninfo $zlib_src_path -o $zlib_info $zlib_obj_path"); +} + +run("$genhtml -o $outdir $infofiles"); + + + +sub run { + my $cmd = shift; + print STDERR "\nrun($cmd)\n" if $verbose > 0; + system("$cmd 2>&1") == 0 or die "\nCan't run \"$cmd\": $?"; +} diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index ac73946dc0..06f0ee1dc6 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -3361,7 +3361,7 @@ dist_ctrl_get_opt(_DHandle, _Opt) -> DHandle :: dist_handle(), InputPackets :: non_neg_integer(), OutputPackets :: non_neg_integer(), - PendingOutputPackets :: boolean(), + PendingOutputPackets :: non_neg_integer(), Res :: {'ok', InputPackets, OutputPackets, PendingOutputPackets}. dist_get_stat(_DHandle) -> diff --git a/erts/vsn.mk b/erts/vsn.mk index 53dce5e815..40a9685e9d 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 10.4.2 +VSN = 10.4.3 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl index a9977b0b1d..afd38dcd08 100644 --- a/lib/compiler/src/beam_ssa.erl +++ b/lib/compiler/src/beam_ssa.erl @@ -96,7 +96,8 @@ %% To avoid the collapsing, change the value of SET_LIMIT to 50 in the %% file erl_types.erl in the hipe application. --type prim_op() :: 'bs_add' | 'bs_extract' | 'bs_init' | 'bs_init_writable' | +-type prim_op() :: 'bs_add' | 'bs_extract' | 'bs_get_tail' | + 'bs_init' | 'bs_init_writable' | 'bs_match' | 'bs_put' | 'bs_start_match' | 'bs_test_tail' | 'bs_utf16_size' | 'bs_utf8_size' | 'build_stacktrace' | 'call' | 'catch_end' | @@ -117,9 +118,10 @@ '+' | '-' | '*' | '/'. %% Primops only used internally during code generation. --type cg_prim_op() :: 'bs_get' | 'bs_match_string' | 'bs_restore' | 'bs_skip' | +-type cg_prim_op() :: 'bs_get' | 'bs_get_position' | 'bs_match_string' | + 'bs_restore' | 'bs_save' | 'bs_set_position' | 'bs_skip' | 'copy' | 'put_tuple_arity' | 'put_tuple_element' | - 'set_tuple_element'. + 'put_tuple_elements' | 'set_tuple_element'. -import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1]). diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index 62cd5b5120..bc28f58712 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -263,7 +263,7 @@ %% @see subtrees/1 %% @see meta/1 --type ctype() :: 'alias' | 'apply' | 'binary' | 'bitrst' | 'call' | 'case' +-type ctype() :: 'alias' | 'apply' | 'binary' | 'bitstr' | 'call' | 'case' | 'catch' | 'clause' | 'cons' | 'fun' | 'let' | 'letrec' | 'literal' | 'map' | 'map_pair' | 'module' | 'primop' | 'receive' | 'seq' | 'try' | 'tuple' | 'values' | 'var'. diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c index 3408ba1b88..941e03cc98 100644 --- a/lib/crypto/c_src/api_ng.c +++ b/lib/crypto/c_src/api_ng.c @@ -100,7 +100,7 @@ static int get_init_args(ErlNifEnv* env, } - if (FORBIDDEN_IN_FIPS(*cipherp)) + if (CIPHER_FORBIDDEN_IN_FIPS(*cipherp)) { *return_term = EXCP_NOTSUP(env, "Forbidden in FIPS"); goto err; @@ -334,12 +334,11 @@ ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg if ((ctx_res = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) return EXCP_ERROR(env, "Can't allocate resource"); - if (!get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], + if (get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], &cipherp, &ret)) - /* Error msg in &ret */ - goto ret; + ret = enif_make_resource(env, ctx_res); + /* else error msg in ret */ - ret = enif_make_resource(env, ctx_res); if(ctx_res) enif_release_resource(ctx_res); } else if (enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) { diff --git a/lib/crypto/c_src/bn.c b/lib/crypto/c_src/bn.c index 34ed4f7ebc..6021d56db6 100644 --- a/lib/crypto/c_src/bn.c +++ b/lib/crypto/c_src/bn.c @@ -32,8 +32,6 @@ int get_bn_from_mpint(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) if (bin.size > INT_MAX - 4) goto err; - ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); - if (bin.size < 4) goto err; sz = (int)bin.size - 4; @@ -60,8 +58,6 @@ int get_bn_from_bin(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) if (bin.size > INT_MAX) goto err; - ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); - if ((ret = BN_bin2bn(bin.data, (int)bin.size, NULL)) == NULL) goto err; @@ -103,8 +99,6 @@ ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) unsigned extra_byte; ERL_NIF_TERM ret; - ASSERT(argc == 4); - if (!get_bn_from_bin(env, argv[0], &bn_base)) goto bad_arg; if (!get_bn_from_bin(env, argv[1], &bn_exponent)) @@ -177,7 +171,6 @@ ERL_NIF_TERM bn2term(ErlNifEnv* env, const BIGNUM *bn) BN_bn2bin(bn, ptr); - ERL_VALGRIND_MAKE_MEM_DEFINED(ptr, dlen); return ret; err: diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index 0532fb7566..e144a891a6 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -214,7 +214,7 @@ ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] if ((cipherp = get_cipher_type_no_key(argv[0])) == NULL) return enif_make_badarg(env); - if (FORBIDDEN_IN_FIPS(cipherp)) + if (CIPHER_FORBIDDEN_IN_FIPS(cipherp)) return enif_raise_exception(env, atom_notsup); if ((cipher = cipherp->cipher.p) == NULL) return enif_raise_exception(env, atom_notsup); @@ -330,7 +330,7 @@ ERL_NIF_TERM cipher_types_as_list(ErlNifEnv* env) for (p = cipher_types; (p->type.atom & (p->type.atom != atom_false)); p++) { if ((prev == p->type.atom) || - FORBIDDEN_IN_FIPS(p) ) + CIPHER_FORBIDDEN_IN_FIPS(p) ) continue; if ((p->cipher.p != NULL) || diff --git a/lib/crypto/c_src/cipher.h b/lib/crypto/c_src/cipher.h index 0e51c410eb..c23e128824 100644 --- a/lib/crypto/c_src/cipher.h +++ b/lib/crypto/c_src/cipher.h @@ -52,10 +52,10 @@ struct cipher_type_t { #ifdef FIPS_SUPPORT /* May have FIPS support, must check dynamically if it is enabled */ -# define FORBIDDEN_IN_FIPS(P) (((P)->flags & NO_FIPS_CIPHER) && FIPS_mode()) +# define CIPHER_FORBIDDEN_IN_FIPS(P) (((P)->flags & NO_FIPS_CIPHER) && FIPS_mode()) #else /* No FIPS support since the symbol FIPS_SUPPORT is undefined */ -# define FORBIDDEN_IN_FIPS(P) 0 +# define CIPHER_FORBIDDEN_IN_FIPS(P) 0 #endif extern ErlNifResourceType* evp_cipher_ctx_rtype; diff --git a/lib/crypto/c_src/digest.c b/lib/crypto/c_src/digest.c index c987a664d5..0f887ab765 100644 --- a/lib/crypto/c_src/digest.c +++ b/lib/crypto/c_src/digest.c @@ -22,7 +22,7 @@ static struct digest_type_t digest_types[] = { - {{"md4"}, + {{"md4"}, NO_FIPS_DIGEST, #ifdef HAVE_MD4 {&EVP_md4} #else @@ -30,7 +30,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"md5"}, + {{"md5"}, NO_FIPS_DIGEST, #ifdef HAVE_MD5 {&EVP_md5} #else @@ -38,7 +38,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"ripemd160"}, + {{"ripemd160"}, NO_FIPS_DIGEST, #ifdef HAVE_RIPEMD160 {&EVP_ripemd160} #else @@ -46,9 +46,9 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha"}, {&EVP_sha1}}, + {{"sha"}, 0, {&EVP_sha1}}, - {{"sha224"}, + {{"sha224"}, 0, #ifdef HAVE_SHA224 {&EVP_sha224} #else @@ -56,7 +56,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha256"}, + {{"sha256"}, 0, #ifdef HAVE_SHA256 {&EVP_sha256} #else @@ -64,7 +64,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha384"}, + {{"sha384"}, 0, #ifdef HAVE_SHA384 {&EVP_sha384} #else @@ -72,7 +72,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha512"}, + {{"sha512"}, 0, #ifdef HAVE_SHA512 {&EVP_sha512} #else @@ -80,7 +80,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha3_224"}, + {{"sha3_224"}, 0, #ifdef HAVE_SHA3_224 {&EVP_sha3_224} #else @@ -88,7 +88,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha3_256"}, + {{"sha3_256"}, 0, #ifdef HAVE_SHA3_256 {&EVP_sha3_256} #else @@ -96,7 +96,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha3_384"}, + {{"sha3_384"}, 0, #ifdef HAVE_SHA3_384 {&EVP_sha3_384} #else @@ -104,7 +104,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"sha3_512"}, + {{"sha3_512"}, 0, #ifdef HAVE_SHA3_512 {&EVP_sha3_512} #else @@ -112,7 +112,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"blake2b"}, + {{"blake2b"}, 0, #ifdef HAVE_BLAKE2 {&EVP_blake2b512} #else @@ -120,7 +120,7 @@ static struct digest_type_t digest_types[] = #endif }, - {{"blake2s"}, + {{"blake2s"}, 0, #ifdef HAVE_BLAKE2 {&EVP_blake2s256} #else @@ -128,7 +128,8 @@ static struct digest_type_t digest_types[] = #endif }, - {{NULL}, {NULL}} + /*==== End of list ==== */ + {{NULL}, 0, {NULL}} }; void init_digest_types(ErlNifEnv* env) diff --git a/lib/crypto/c_src/digest.h b/lib/crypto/c_src/digest.h index 06852416cf..b1f8128a1f 100644 --- a/lib/crypto/c_src/digest.h +++ b/lib/crypto/c_src/digest.h @@ -28,12 +28,25 @@ struct digest_type_t { const char* str; /* before init, NULL for end-of-table */ ERL_NIF_TERM atom; /* after init, 'false' for end-of-table */ }type; + unsigned flags; union { const EVP_MD* (*funcp)(void); /* before init, NULL if notsup */ const EVP_MD* p; /* after init, NULL if notsup */ }md; }; +/* masks in the flags field if digest_type_t */ +#define NO_FIPS_DIGEST 1 + +#ifdef FIPS_SUPPORT +/* May have FIPS support, must check dynamically if it is enabled */ +# define DIGEST_FORBIDDEN_IN_FIPS(P) (((P)->flags & NO_FIPS_DIGEST) && FIPS_mode()) +#else +/* No FIPS support since the symbol FIPS_SUPPORT is undefined */ +# define DIGEST_FORBIDDEN_IN_FIPS(P) 0 +#endif + + void init_digest_types(ErlNifEnv* env); struct digest_type_t* get_digest_type(ERL_NIF_TERM type); diff --git a/lib/crypto/c_src/mac.c b/lib/crypto/c_src/mac.c index 8b2710b91a..149975ba9d 100644 --- a/lib/crypto/c_src/mac.c +++ b/lib/crypto/c_src/mac.c @@ -34,6 +34,7 @@ struct mac_type_t { const char* str; /* before init, NULL for end-of-table */ ERL_NIF_TERM atom; /* after init, 'false' for end-of-table */ }name; + unsigned flags; union { const int pkey_type; }alg; @@ -41,6 +42,9 @@ struct mac_type_t { size_t key_len; /* != 0 to also match on key_len */ }; +/* masks in the flags field if mac_type_t */ +#define NO_FIPS_MAC 1 + #define NO_mac 0 #define HMAC_mac 1 #define CMAC_mac 2 @@ -48,7 +52,7 @@ struct mac_type_t { static struct mac_type_t mac_types[] = { - {{"poly1305"}, + {{"poly1305"}, NO_FIPS_MAC, #ifdef HAVE_POLY1305 /* If we have POLY then we have EVP_PKEY */ {EVP_PKEY_POLY1305}, POLY1305_mac, 32 @@ -57,7 +61,7 @@ static struct mac_type_t mac_types[] = #endif }, - {{"hmac"}, + {{"hmac"}, 0, #ifdef HAS_EVP_PKEY_CTX {EVP_PKEY_HMAC}, HMAC_mac, 0 #else @@ -66,7 +70,7 @@ static struct mac_type_t mac_types[] = #endif }, - {{"cmac"}, + {{"cmac"}, 0, #ifdef HAVE_CMAC /* If we have CMAC then we have EVP_PKEY */ {EVP_PKEY_CMAC}, CMAC_mac, 0 @@ -76,12 +80,21 @@ static struct mac_type_t mac_types[] = }, /*==== End of list ==== */ - {{NULL}, + {{NULL}, 0, {0}, NO_mac, 0 } }; +#ifdef FIPS_SUPPORT +/* May have FIPS support, must check dynamically if it is enabled */ +# define MAC_FORBIDDEN_IN_FIPS(P) (((P)->flags & NO_FIPS_MAC) && FIPS_mode()) +#else +/* No FIPS support since the symbol FIPS_SUPPORT is undefined */ +# define MAC_FORBIDDEN_IN_FIPS(P) 0 +#endif + + /*************************** Mandatory prototypes ***************************/ @@ -219,6 +232,12 @@ ERL_NIF_TERM mac_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) goto err; } + if (MAC_FORBIDDEN_IN_FIPS(macp)) + { + return_term = EXCP_NOTSUP(env, "MAC algorithm forbidden in FIPS"); + goto err; + } + /*-------------------------------------------------- Algorithm dependent indata checking and computation. If EVP_PKEY is available, only set the pkey variable @@ -245,7 +264,11 @@ ERL_NIF_TERM mac_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return_term = EXCP_NOTSUP(env, "Unsupported digest algorithm"); goto err; } - + if (DIGEST_FORBIDDEN_IN_FIPS(digp)) + { + return_term = EXCP_NOTSUP(env, "Digest algorithm for HMAC forbidden in FIPS"); + goto err; + } md = digp->md.p; #ifdef HAS_EVP_PKEY_CTX @@ -284,7 +307,7 @@ ERL_NIF_TERM mac_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) goto err; } - if (FORBIDDEN_IN_FIPS(cipherp)) + if (CIPHER_FORBIDDEN_IN_FIPS(cipherp)) { return_term = EXCP_NOTSUP(env, "Cipher algorithm not supported in FIPS"); goto err; @@ -496,6 +519,12 @@ ERL_NIF_TERM mac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) goto err; } + if (MAC_FORBIDDEN_IN_FIPS(macp)) + { + return_term = EXCP_NOTSUP(env, "MAC algorithm forbidden in FIPS"); + goto err; + } + /*-------------------------------------------------- Algorithm dependent indata checking and computation. If EVP_PKEY is available, only set the pkey variable @@ -522,7 +551,11 @@ ERL_NIF_TERM mac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return_term = EXCP_NOTSUP(env, "Unsupported digest algorithm"); goto err; } - + if (DIGEST_FORBIDDEN_IN_FIPS(digp)) + { + return_term = EXCP_NOTSUP(env, "Digest algorithm for HMAC forbidden in FIPS"); + goto err; + } md = digp->md.p; # ifdef HAVE_PKEY_new_raw_private_key @@ -553,7 +586,7 @@ ERL_NIF_TERM mac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) goto err; } - if (FORBIDDEN_IN_FIPS(cipherp)) + if (CIPHER_FORBIDDEN_IN_FIPS(cipherp)) { return_term = EXCP_NOTSUP(env, "Cipher algorithm not supported in FIPS"); goto err; diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 8988a18482..3973cf3f9f 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -2016,7 +2016,7 @@ FloatValue = rand:uniform(). % again <seealso marker="#stream_encrypt-2">stream_encrypt</seealso> and <seealso marker="#stream_decrypt-2">stream_decrypt</seealso></p> <p>For keylengths see the - <seealso marker="crypto:algorithm_details#stream-ciphers">User's Guide</seealso>. + <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. </p> </desc> </func> @@ -2032,7 +2032,7 @@ FloatValue = rand:uniform(). % again <seealso marker="#stream_encrypt-2">stream_encrypt</seealso> and <seealso marker="#stream_decrypt-2">stream_decrypt</seealso>.</p> <p>For keylengths and iv-sizes see the - <seealso marker="crypto:algorithm_details#stream-ciphers">User's Guide</seealso>. + <seealso marker="crypto:algorithm_details#ciphers">User's Guide</seealso>. </p> </desc> </func> diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index d4fe064edd..c1bc5ff597 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -605,11 +605,9 @@ ordinal(N) when is_integer(N) -> io_lib:format("~wth", [N]). %% Functions that parse type strings, literal strings, and contract %% strings. Return strings formatted by erl_pp. -%% If lib/hipe/cerl/erl_types.erl is compiled with DEBUG=true, -%% the contents of opaque types are showed inside brackets. -%% Since erl_parse:parse_form() cannot handle the bracket syntax, -%% no indentation is done. -%%-define(DEBUG , true). +%% Note we always have to catch any error when trying to parse +%% the syntax because other BEAM languages may not emit an +%% Erlang AST that transforms into valid Erlang Source Code. -define(IND, 10). @@ -620,13 +618,15 @@ con(M, F, Src, I) -> sig(Src, false) -> Src; sig(Src, true) -> - Str = lists:flatten(io_lib:format("-spec ~w:~tw~ts.", [a, b, Src])), - {ok, Tokens, _EndLocation} = erl_scan:string(Str), - exec(fun() -> - {ok, {attribute, _, spec, {_MFA, Types}}} = - erl_parse:parse_form(Tokens), - indentation(?IND) ++ pp_spec(Types) - end, Src). + try + Str = lists:flatten(io_lib:format("-spec ~w:~tw~ts.", [a, b, Src])), + {ok, Tokens, _EndLocation} = erl_scan:string(Str), + {ok, {attribute, _, spec, {_MFA, Types}}} = + erl_parse:parse_form(Tokens), + indentation(?IND) ++ pp_spec(Types) + catch + _:_ -> Src + end. %% Argument(list)s are a mix of types and Erlang code. Note: sometimes %% (contract_range, call_without_opaque, opaque_type_test), the initial @@ -681,21 +681,15 @@ ts(Src) -> [C1|Src1] = Src, % $< (product) or $( (arglist) [C2|RevSrc2] = lists:reverse(Src1), Src2 = lists:reverse(RevSrc2), - exec(fun() -> - Types = parse_types_and_literals(Src2), - CommaInd = [$, | Ind], - (indentation(?IND-1) ++ - [C1 | lists:join(CommaInd, [pp_type(Type) || Type <- Types])] ++ - [C2]) - end, Src). - --ifdef(DEBUG). -exec(F, R) -> - try F() catch _:_ -> R end. --else. -exec(F, _) -> - F(). --endif. + try + Types = parse_types_and_literals(Src2), + CommaInd = [$, | Ind], + (indentation(?IND-1) ++ + [C1 | lists:join(CommaInd, [pp_type(Type) || Type <- Types])] ++ + [C2]) + catch + _:_ -> Src + end. indentation(I) -> [$\n | lists:duplicate(I, $\s)]. diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c index a94d6e2ad8..aee7f7eeb0 100644 --- a/lib/erl_interface/src/misc/ei_printterm.c +++ b/lib/erl_interface/src/misc/ei_printterm.c @@ -124,6 +124,7 @@ static int print_term(FILE* fp, ei_x_buff* x, erlang_fun fun; double d; long l; + const char* delim; int tindex = *index; @@ -240,41 +241,47 @@ static int print_term(FILE* fp, ei_x_buff* x, m = BINPRINTSIZE; else m = l; - --m; + delim = ""; for (i = 0; i < m; ++i) { - ch_written += xprintf(fp, x, "%d,", p[i]); + ch_written += xprintf(fp, x, "%s%u", delim, (unsigned char)p[i]); + delim = ","; } - ch_written += xprintf(fp, x, "%d", p[i]); if (l > BINPRINTSIZE) ch_written += xprintf(fp, x, ",..."); xputc('>', fp, x); ++ch_written; ei_free(p); break; case ERL_BIT_BINARY_EXT: { - const char* cp; + const unsigned char* cp; size_t bits; unsigned int bitoffs; int trunc = 0; - if (ei_decode_bitstring(buf, index, &cp, &bitoffs, &bits) < 0 + if (ei_decode_bitstring(buf, index, (const char**)&cp, &bitoffs, &bits) < 0 || bitoffs != 0) { goto err; } ch_written += xprintf(fp, x, "#Bits<"); - m = (bits+7) / 8; - if (m > BINPRINTSIZE) { + if ((bits+7) / 8 > BINPRINTSIZE) { m = BINPRINTSIZE; trunc = 1; } - --m; + else + m = bits / 8; + + delim = ""; for (i = 0; i < m; ++i) { - ch_written += xprintf(fp, x, "%d,", cp[i]); + ch_written += xprintf(fp, x, "%s%u", delim, cp[i]); + delim = ","; } - ch_written += xprintf(fp, x, "%d", cp[i]); if (trunc) ch_written += xprintf(fp, x, ",..."); - else if (bits % 8 != 0) - ch_written += xprintf(fp, x, ":%u", (unsigned)(bits % 8)); + else { + bits %= 8; + if (bits) + ch_written += xprintf(fp, x, "%s%u:%u", delim, + (cp[i] >> (8-bits)), bits); + } xputc('>', fp, x); ++ch_written; break; } diff --git a/lib/erl_interface/test/ei_print_SUITE.erl b/lib/erl_interface/test/ei_print_SUITE.erl index f2a2548183..8a35b22ae5 100644 --- a/lib/erl_interface/test/ei_print_SUITE.erl +++ b/lib/erl_interface/test/ei_print_SUITE.erl @@ -27,7 +27,7 @@ -export([all/0, suite/0, init_per_testcase/2, atoms/1, tuples/1, lists/1, strings/1, - maps/1, funs/1]). + maps/1, funs/1, binaries/1, bitstrings/1]). -import(runner, [get_term/1]). @@ -38,7 +38,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [atoms, tuples, lists, strings, maps, funs]. + [atoms, tuples, lists, strings, maps, funs, binaries, bitstrings]. init_per_testcase(Case, Config) -> runner:init_per_testcase(?MODULE, Case, Config). @@ -163,3 +163,44 @@ funs(Config) -> runner:recv_eot(P), ok. + +binaries(Config) -> + P = runner:start(Config, ?binaries), + + "#Bin<>" = send_term_get_printed(P, <<>>), + "#Bin<1,2,3>" = send_term_get_printed(P, <<1,2,3>>), + "#Bin<0,127,128,255>" = send_term_get_printed(P, <<0,127,128,255>>), + Bin30 = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30>>, + "#Bin<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30>" + = send_term_get_printed(P, Bin30), + "#Bin<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,...>" + = send_term_get_printed(P, <<Bin30/binary,31>>), + + runner:recv_eot(P), + ok. + +bitstrings(Config) -> + P = runner:start(Config, ?bitstrings), + + "#Bits<1:1>" = send_term_get_printed(P, <<1:1>>), + "#Bits<123:7>" = send_term_get_printed(P, <<123:7>>), + "#Bits<1,2,3:4>" = send_term_get_printed(P, <<1,2,3:4>>), + "#Bits<0,127,128,255,126:7>" = send_term_get_printed(P, <<0,127,128,255,-2:7>>), + Bits30 = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19, + 20,21,22,23,24,25,26,27,28,29,30:5>>, + "#Bits<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30:5>" + = send_term_get_printed(P, Bits30), + "#Bin<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,241>" + = send_term_get_printed(P, <<Bits30/bits,1:3>>), + "#Bits<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,240,...>" + = send_term_get_printed(P, <<Bits30/bits,1:4>>), + + runner:recv_eot(P), + ok. + + + +send_term_get_printed(Port, Term) -> + Port ! {self(), {command, term_to_binary(Term)}}, + {term, String} = get_term(Port), + String. diff --git a/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c b/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c index 0e2b24e45a..27d4153250 100644 --- a/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c +++ b/lib/erl_interface/test/ei_print_SUITE_data/ei_print_test.c @@ -288,3 +288,64 @@ TESTCASE(funs) report(1); } + + +TESTCASE(binaries) +{ + char *buf; + long len; + int err, n, index; + ei_x_buff x; + + ei_init(); + + for (n = 5; n; n--) { + buf = read_packet(NULL); + + index = 0; + err = ei_decode_version(buf, &index, NULL); + if (err != 0) + fail1("ei_decode_version returned %d", err); + err = ei_decode_binary(buf, &index, NULL, &len); + if (err != 0) + fail1("ei_decode_binary returned %d", err); + + ei_x_new(&x); + ei_x_append_buf(&x, buf, index); + send_printed_buf(&x); + ei_x_free(&x); + + free_packet(buf); + } + report(1); +} + +TESTCASE(bitstrings) +{ + char *buf; + long len; + int err, n, index; + ei_x_buff x; + + ei_init(); + + for (n = 7; n; n--) { + buf = read_packet(NULL); + + index = 0; + err = ei_decode_version(buf, &index, NULL); + if (err != 0) + fail1("ei_decode_version returned %d", err); + err = ei_decode_bitstring(buf, &index, NULL, NULL, NULL); + if (err != 0) + fail1("ei_decode_bitstring returned %d", err); + + ei_x_new(&x); + ei_x_append_buf(&x, buf, index); + send_printed_buf(&x); + ei_x_free(&x); + + free_packet(buf); + } + report(1); +} diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index 1aa4b7a3a2..5aa2caadf0 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -89,8 +89,8 @@ <p>Filter functions can be used for more sophisticated filtering than the log level check provides. A filter function can stop or pass a log event, based on any of the event's contents. It can - also modify all parts of the log event. See see - section <seealso marker="#filters">Filters</seealso> for more + also modify all parts of the log event. See section + <seealso marker="#filters">Filters</seealso> for more details.</p> <p>If a log event passes through all primary filters and all handler filters for a specific handler, Logger forwards the diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index e140d2ba30..4d31eeea3d 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,24 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p><c>user</c>/<c>user_drv</c> could respond to io + requests before they had been processed, which could + cause data to be dropped if the emulator was halted soon + after a call to <c>io:format/2</c>, such as in an + escript.</p> + <p> + Own Id: OTP-15805</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index cd0397a98c..95853a7a8f 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -38,7 +38,9 @@ {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.3$">>,[restart_new_emulator]}, {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.4$">>,[restart_new_emulator]}, + {<<"^6\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^6\\.0$">>,[restart_new_emulator]}, {<<"^6\\.0\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -50,4 +52,6 @@ {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.3$">>,[restart_new_emulator]}, {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.4$">>,[restart_new_emulator]}, + {<<"^6\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 765e890157..e5188aa9b5 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.4 +KERNEL_VSN = 6.4.1 diff --git a/lib/os_mon/src/disksup.erl b/lib/os_mon/src/disksup.erl index 5118d807e1..4253067c90 100644 --- a/lib/os_mon/src/disksup.erl +++ b/lib/os_mon/src/disksup.erl @@ -264,7 +264,7 @@ check_disk_space({unix, irix}, Port, Threshold) -> Result = my_cmd("/usr/sbin/df -lk",Port), check_disks_irix(skip_to_eol(Result), Threshold); check_disk_space({unix, linux}, Port, Threshold) -> - Result = my_cmd("/bin/df -lk", Port), + Result = my_cmd("/bin/df -lk -x squashfs", Port), check_disks_solaris(skip_to_eol(Result), Threshold); check_disk_space({unix, posix}, Port, Threshold) -> Result = my_cmd("df -k -P", Port), diff --git a/lib/snmp/src/agent/snmpa_set.erl b/lib/snmp/src/agent/snmpa_set.erl index 9833d6fdcc..b3a3bf0ab0 100644 --- a/lib/snmp/src/agent/snmpa_set.erl +++ b/lib/snmp/src/agent/snmpa_set.erl @@ -163,10 +163,14 @@ set_phase_two(MyVarbinds, SubagentVarbinds) -> [MyVarbinds, SubagentVarbinds]), case snmpa_set_lib:try_set(MyVarbinds) of {noError, 0} -> + ?vtrace("set phase two: (local) varbinds set ok", []), set_phase_two_subagents(SubagentVarbinds); - {ErrorStatus, Index} -> + {ErrorStatus, ErrorIndex} -> + ?vlog("set phase two: (local) varbinds set failed" + "~n ErrorStatus: ~p" + "~n ErrorIndex: ~p", [ErrorStatus, ErrorIndex]), set_phase_two_undo_subagents(SubagentVarbinds), - {ErrorStatus, Index} + {ErrorStatus, ErrorIndex} end. %%----------------------------------------------------------------- @@ -188,6 +192,7 @@ set_phase_two_subagents([{SubAgentPid, SAVbs} | SubagentVarbinds]) -> {_SAOids, Vbs} = sa_split(SAVbs), case catch snmpa_agent:subagent_set(SubAgentPid, [phase_two, set, Vbs]) of {noError, 0} -> + ?vtrace("set phase two: subagent ~p varbinds set ok", [SubAgentPid]), set_phase_two_subagents(SubagentVarbinds); {'EXIT', Reason} -> user_err("Lost contact with subagent (set)~n~w. Using genErr", @@ -195,10 +200,14 @@ set_phase_two_subagents([{SubAgentPid, SAVbs} | SubagentVarbinds]) -> set_phase_two_undo_subagents(SubagentVarbinds), {genErr, 0}; {ErrorStatus, ErrorIndex} -> + ?vlog("set phase two: subagent ~p varbinds set failed" + "~n ErrorStatus: ~p" + "~n ErrorIndex: ~p", [SubAgentPid, ErrorStatus, ErrorIndex]), set_phase_two_undo_subagents(SubagentVarbinds), {ErrorStatus, ErrorIndex} end; set_phase_two_subagents([]) -> + ?vtrace("set phase two: subagent(s) set ok", []), {noError, 0}. %%----------------------------------------------------------------- diff --git a/lib/snmp/test/snmp_agent_test.erl b/lib/snmp/test/snmp_agent_test.erl index 71e3fa3b9a..860ca17cdb 100644 --- a/lib/snmp/test/snmp_agent_test.erl +++ b/lib/snmp/test/snmp_agent_test.erl @@ -667,22 +667,39 @@ init_per_group(GroupName, Config) -> snmp_test_lib:init_group_top_dir(GroupName, Config). init_per_group_ipv6(GroupName, Config, Init) -> - {ok, Hostname0} = inet:gethostname(), - case ct:require(ipv6_hosts) of - ok -> - case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of - true -> - Init( - snmp_test_lib:init_group_top_dir( - GroupName, - [{ipfamily, inet6}, - {ip, ?LOCALHOST(inet6)} - | lists:keydelete(ip, 1, Config)])); - false -> - {skip, "Host does not support IPV6"} - end; - _ -> - {skip, "Test config ipv6_hosts is missing"} + %% <OS-CONDITIONAL-SKIP> + %% This is a higly questionable test. + %% But until we have time to figure out what IPv6 issues + %% are actually causing the failures... + OSSkipable = [{unix, + [ + {darwin, fun(V) when (V > {9, 8, 0}) -> + %% This version is OK: No Skip + false; + (_) -> + %% This version is *not* ok: Skip + true + end} + ] + }], + %% </OS-CONDITIONAL-SKIP> + case ?OS_BASED_SKIP(OSSkipable) of + true -> + {skip, "Host *may* not *properly* support IPV6"}; + false -> + %% Even if this host supports IPv6 we don't use it unless its + %% one of the configured/supported IPv6 hosts... + case (?HAS_SUPPORT_IPV6() andalso ?IS_IPV6_HOST()) of + true -> + Init( + snmp_test_lib:init_group_top_dir( + GroupName, + [{ipfamily, inet6}, + {ip, ?LOCALHOST(inet6)} + | lists:keydelete(ip, 1, Config)])); + false -> + {skip, "Host does not support IPv6"} + end end. end_per_group(all_tcs, Config) -> diff --git a/lib/snmp/test/snmp_agent_test_lib.erl b/lib/snmp/test/snmp_agent_test_lib.erl index 6defdadb5a..1128fc8a8c 100644 --- a/lib/snmp/test/snmp_agent_test_lib.erl +++ b/lib/snmp/test/snmp_agent_test_lib.erl @@ -66,7 +66,7 @@ ]). %% Internal exports --export([wait/5, run/4]). +-export([tc_wait/5, tc_run/4]). -include_lib("kernel/include/file.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -276,36 +276,94 @@ init_case(Config) when is_list(Config) -> %%% configuration. %%%-------------------------------------------------- -try_test(Mod, Func) -> - call(get(mgr_node), ?MODULE, run, [Mod, Func, [], []]). - -try_test(Mod, Func, A) -> - call(get(mgr_node), ?MODULE, run, [Mod, Func, A, []]). - -try_test(Mod, Func, A, Opts) -> - call(get(mgr_node), ?MODULE, run, [Mod, Func, A, Opts]). - -call(N,M,F,A) -> - ?DBG("call -> entry with~n" - " N: ~p~n" - " M: ~p~n" - " F: ~p~n" - " A: ~p~n" - " when~n" - " get(): ~p", - [N,M,F,A,get()]), - spawn(N, ?MODULE, wait, [self(),get(),M,F,A]), +try_test(TcRunMod, TcRunFunc) -> + try_test(TcRunMod, TcRunFunc, []). + +try_test(TcRunMod, TcRunFunc, TcRunArgs) -> + try_test(TcRunMod, TcRunFunc, TcRunArgs, []). + +try_test(TcRunMod, TcRunFunc, TcRunArgs, TcRunOpts) -> + Node = get(mgr_node), + Mod = ?MODULE, + Func = tc_run, + Args = [TcRunMod, TcRunFunc, TcRunArgs, TcRunOpts], + tc_try(Node, Mod, Func, Args). + +%% We spawn a test case runner process on the manager node. +%% The assumption is that the manager shall do something, but +%% not all test cases have the manager perform actions. +%% In some cases we make a rpc call back to the agent node directly +%% and call something in the agent... (for example the info_test +%% test case). +%% We should use link (instead of monitor) in order for the test case +%% timeout cleanup (kills) should have effect on the test case runner +%% process as well. + +tc_try(N, M, F, A) -> + ?PRINT2("tc_try -> entry with" + "~n N: ~p" + "~n M: ~p" + "~n F: ~p" + "~n A: ~p" + "~n when" + "~n get(): ~p" + "~n", [N, + M, F, A, + get()]), + case net_adm:ping(N) of + pong -> + ?PRINT2("tc_try -> ~p still running - start runner~n", [N]), + OldFlag = trap_exit(true), % Make sure we catch it + Runner = spawn_link(N, ?MODULE, tc_wait, [self(), get(), M, F, A]), + await_tc_runner_started(Runner, OldFlag), + await_tc_runner_done(Runner, OldFlag); + pang -> + ?EPRINT2("tc_try -> ~p *not* running~n", [N]), + exit({node_not_running, N}) + end. + +await_tc_runner_started(Runner, OldFlag) -> + ?PRINT2("await tc-runner (~p) start ack~n", [Runner]), + receive + {'EXIT', Runner, Reason} -> + ?EPRINT2("TC runner start failed: " + "~n ~p~n", [Reason]), + exit({tx_runner_start_failed, Reason}); + {tc_runner_started, Runner} -> + ?PRINT2("TC runner start acknowledged~n"), + ok + after 10000 -> + trap_exit(OldFlag), + unlink_and_flush_exit(Runner), + RunnerInfo = process_info(Runner), + ?EPRINT2("TC runner start timeout: " + "~n ~p", [RunnerInfo]), + %% If we don't get a start ack within 10 seconds, we are f*ed + exit(Runner, kill), + exit({tc_runner_start, timeout, RunnerInfo}) + end. + +await_tc_runner_done(Runner, OldFlag) -> receive - {done, {'EXIT', Rn}, Loc} -> - ?DBG("call -> done with exit: " - "~n Rn: ~p" - "~n Loc: ~p", [Rn, Loc]), + {'EXIT', Runner, Reason} -> + ?EPRINT2("TC runner failed: " + "~n ~p~n", [Reason]), + exit({tx_runner_failed, Reason}); + {tc_runner_done, Runner, {'EXIT', Rn}, Loc} -> + ?PRINT2("call -> done with exit: " + "~n Rn: ~p" + "~n Loc: ~p" + "~n", [Rn, Loc]), + trap_exit(OldFlag), + unlink_and_flush_exit(Runner), put(test_server_loc, Loc), exit(Rn); - {done, Ret, _Zed} -> + {tc_runner_done, Runner, Ret, _Zed} -> ?DBG("call -> done:" "~n Ret: ~p" "~n Zed: ~p", [Ret, _Zed]), + trap_exit(OldFlag), + unlink_and_flush_exit(Runner), case Ret of {error, Reason} -> exit(Reason); @@ -314,49 +372,66 @@ call(N,M,F,A) -> end end. -wait(From, Env, M, F, A) -> - ?DBG("wait -> entry with" - "~n From: ~p" - "~n Env: ~p" - "~n M: ~p" - "~n F: ~p" - "~n A: ~p", [From, Env, M, F, A]), +trap_exit(Flag) when is_boolean(Flag) -> + erlang:process_flag(trap_exit, Flag). + +unlink_and_flush_exit(Pid) -> + unlink(Pid), + receive + {'EXIT', Pid, _} -> + ok + after 0 -> + ok + end. + +tc_wait(From, Env, M, F, A) -> + ?PRINT2("tc_wait -> entry with" + "~n From: ~p" + "~n Env: ~p" + "~n M: ~p" + "~n F: ~p" + "~n A: ~p", [From, Env, M, F, A]), + From ! {tc_runner_started, self()}, lists:foreach(fun({K,V}) -> put(K,V) end, Env), - Rn = (catch apply(M, F, A)), - ?DBG("wait -> Rn: ~n~p", [Rn]), - From ! {done, Rn, get(test_server_loc)}, - exit(Rn). - -run(Mod, Func, Args, Opts) -> - ?DBG("run -> entry with" - "~n Mod: ~p" - "~n Func: ~p" - "~n Args: ~p" - "~n Opts: ~p", [Mod, Func, Args, Opts]), - M = get(mib_dir), - Dir = get(mgr_dir), - User = snmp_misc:get_option(user, Opts, "all-rights"), - SecLevel = snmp_misc:get_option(sec_level, Opts, noAuthNoPriv), - EngineID = snmp_misc:get_option(engine_id, Opts, "agentEngine"), + ?PRINT2("tc_wait -> env set - now run tc~n"), + Res = (catch apply(M, F, A)), + ?PRINT2("tc_wait -> tc run done: " + "~n ~p" + "~n", [Res]), + From ! {tc_runner_done, self(), Res, get(test_server_loc)}, + exit(Res). + +tc_run(Mod, Func, Args, Opts) -> + ?PRINT2("tc_run -> entry with" + "~n Mod: ~p" + "~n Func: ~p" + "~n Args: ~p" + "~n Opts: ~p" + "~n", [Mod, Func, Args, Opts]), + (catch snmp_test_mgr:stop()), % If we had a running mgr from a failed case + M = get(mib_dir), + Dir = get(mgr_dir), + User = snmp_misc:get_option(user, Opts, "all-rights"), + SecLevel = snmp_misc:get_option(sec_level, Opts, noAuthNoPriv), + EngineID = snmp_misc:get_option(engine_id, Opts, "agentEngine"), CtxEngineID = snmp_misc:get_option(context_engine_id, Opts, EngineID), - Community = snmp_misc:get_option(community, Opts, "all-rights"), - ?DBG("run -> start crypto app",[]), - _CryptoRes = ?CRYPTO_START(), - ?DBG("run -> Crypto: ~p", [_CryptoRes]), - catch snmp_test_mgr:stop(), % If we had a running mgr from a failed case - StdM = join(code:priv_dir(snmp), "mibs") ++ "/", - Vsn = get(vsn), - ?DBG("run -> config:" - "~n M: ~p" - "~n Vsn: ~p" - "~n Dir: ~p" - "~n User: ~p" - "~n SecLevel: ~p" - "~n EngineID: ~p" - "~n CtxEngineID: ~p" - "~n Community: ~p" - "~n StdM: ~p", - [M,Vsn,Dir,User,SecLevel,EngineID,CtxEngineID,Community,StdM]), + Community = snmp_misc:get_option(community, Opts, "all-rights"), + ?DBG("tc_run -> start crypto app",[]), + _CryptoRes = ?CRYPTO_START(), + ?DBG("tc_run -> Crypto: ~p", [_CryptoRes]), + StdM = join(code:priv_dir(snmp), "mibs") ++ "/", + Vsn = get(vsn), + ?PRINT2("tc_run -> config:" + "~n M: ~p" + "~n Vsn: ~p" + "~n Dir: ~p" + "~n User: ~p" + "~n SecLevel: ~p" + "~n EngineID: ~p" + "~n CtxEngineID: ~p" + "~n Community: ~p" + "~n StdM: ~p" + "~n", [M,Vsn,Dir,User,SecLevel,EngineID,CtxEngineID,Community,StdM]), case snmp_test_mgr:start([%% {agent, snmp_test_lib:hostname()}, {packet_server_debug, true}, {debug, true}, @@ -377,23 +452,23 @@ run(Mod, Func, Args, Opts) -> {ok, _Pid} -> case (catch apply(Mod, Func, Args)) of {'EXIT', Reason} -> - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), ?FAIL({apply_failed, {Mod, Func, Args}, Reason}); Res -> - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), Res end; {error, Reason} -> ?EPRINT2("Failed starting (test) manager: " "~n ~p", [Reason]), - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), ?line ?FAIL({mgr_start_error, Reason}); Err -> ?EPRINT2("Failed starting (test) manager: " "~n ~p", [Err]), - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), ?line ?FAIL({mgr_start_failure, Err}) end. diff --git a/lib/snmp/test/snmp_manager_test.erl b/lib/snmp/test/snmp_manager_test.erl index 5b0ebf8647..d959d9e09b 100644 --- a/lib/snmp/test/snmp_manager_test.erl +++ b/lib/snmp/test/snmp_manager_test.erl @@ -619,38 +619,47 @@ init_per_group(event_tests_mt = GroupName, Config) -> GroupName, [{manager_net_if_module, snmpm_net_if_mt} | Config]); init_per_group(ipv6_mt = GroupName, Config) -> - {ok, Hostname0} = inet:gethostname(), - case ct:require(ipv6_hosts) of - ok -> - case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of - true -> - ipv6_init( - snmp_test_lib:init_group_top_dir( - GroupName, - [{manager_net_if_module, snmpm_net_if_mt} - | Config])); - false -> - {skip, "Host does not support IPv6"} - end; - _ -> - {skip, "Test config ipv6_hosts is missing"} - end; + init_per_group_ipv6(GroupName, + [{manager_net_if_module, snmpm_net_if_mt} | Config]); init_per_group(ipv6 = GroupName, Config) -> - {ok, Hostname0} = inet:gethostname(), - case ct:require(ipv6_hosts) of - ok -> - case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of - true -> - ipv6_init(snmp_test_lib:init_group_top_dir(GroupName, Config)); - false -> - {skip, "Host does not support IPv6"} - end; - _ -> - {skip, "Test config ipv6_hosts is missing"} - end; + init_per_group_ipv6(GroupName, Config); init_per_group(GroupName, Config) -> snmp_test_lib:init_group_top_dir(GroupName, Config). + +init_per_group_ipv6(GroupName, Config) -> + %% <OS-CONDITIONAL-SKIP> + OSSkipable = [{unix, + [ + {darwin, fun(V) when (V > {9, 8, 0}) -> + %% This version is OK: No Skip + false; + (_) -> + %% This version is *not* ok: Skip + %% We need a fully qualified hostname + %% to get a proper IPv6 address (in this + %% version), but its just to messy, so + %% instead we skip this **OLD** darwin... + true + end} + ] + }], + %% </OS-CONDITIONAL-SKIP> + case ?OS_BASED_SKIP(OSSkipable) of + true -> + {skip, "Host *may* not *properly* support IPV6"}; + false -> + %% Even if this host supports IPv6 we don't use it unless its + %% one of the configures/supported IPv6 hosts... + case (?HAS_SUPPORT_IPV6() andalso ?IS_IPV6_HOST()) of + true -> + ipv6_init(snmp_test_lib:init_group_top_dir(GroupName, Config)); + false -> + {skip, "Host does not support IPv6"} + end + end. + + end_per_group(_GroupName, Config) -> %% Do we really need to do this? lists:keydelete(snmp_group_top_dir, 1, Config). diff --git a/lib/snmp/test/snmp_test_lib.erl b/lib/snmp/test/snmp_test_lib.erl index a483690653..42f710e4cd 100644 --- a/lib/snmp/test/snmp_test_lib.erl +++ b/lib/snmp/test/snmp_test_lib.erl @@ -25,7 +25,9 @@ -export([hostname/0, hostname/1, localhost/0, localhost/1, os_type/0, sz/1, display_suite_info/1]). --export([non_pc_tc_maybe_skip/4, os_based_skip/1]). +-export([non_pc_tc_maybe_skip/4, os_based_skip/1, + has_support_ipv6/0, has_support_ipv6/1, + is_ipv6_host/0, is_ipv6_host/1]). -export([fix_data_dir/1, init_suite_top_dir/2, init_group_top_dir/2, init_testcase_top_dir/2, lookup/2, @@ -52,11 +54,12 @@ hostname() -> hostname(node()). hostname(Node) -> - from($@, atom_to_list(Node)). - -from(H, [H | T]) -> T; -from(H, [_ | T]) -> from(H, T); -from(_H, []) -> []. + case string:tokens(atom_to_list(Node), [$@]) of + [_, Host] -> + Host; + _ -> + [] + end. %% localhost() -> %% {ok, Ip} = snmp_misc:ip(net_adm:localhost()), @@ -78,7 +81,9 @@ localhost(Family) -> {error, Reason1} -> fail({getifaddrs, Reason1}, ?MODULE, ?LINE) end; - {ok, {0, _, _, _, _, _, _, _}} when (Family =:= inet6) -> + {ok, {A1, _, _, _, _, _, _, _}} when (Family =:= inet6) andalso + ((A1 =:= 0) orelse + (A1 =:= 16#fe80)) -> %% Ouch, we need to use something else case inet:getifaddrs() of {ok, IfList} -> @@ -207,53 +212,108 @@ non_pc_tc_maybe_skip(Config, Condition, File, Line) end. +%% The type and spec'ing is just to increase readability +-type os_family() :: win32 | unix. +-type os_name() :: atom(). +-type os_version() :: string() | {non_neg_integer(), + non_neg_integer(), + non_neg_integer()}. +-type os_skip_check() :: fun(() -> boolean()) | + fun((os_version()) -> boolean()). +-type skippable() :: any | [os_family() | + {os_family(), os_name() | + [os_name() | {os_name(), + os_skip_check()}]}]. + +-spec os_based_skip(skippable()) -> boolean(). + os_based_skip(any) -> - io:format("os_based_skip(any) -> entry" - "~n", []), true; os_based_skip(Skippable) when is_list(Skippable) -> - io:format("os_based_skip -> entry with" - "~n Skippable: ~p" - "~n", [Skippable]), - {OsFam, OsName} = - case os:type() of - {_Fam, _Name} = FamAndName -> - FamAndName; - Fam -> - {Fam, undefined} - end, - io:format("os_based_skip -> os-type: " - "~n OsFam: ~p" - "~n OsName: ~p" - "~n", [OsFam, OsName]), + os_base_skip(Skippable, os:type()); +os_based_skip(_Crap) -> + false. + +os_base_skip(Skippable, {OsFam, OsName}) -> + os_base_skip(Skippable, OsFam, OsName); +os_base_skip(Skippable, OsFam) -> + os_base_skip(Skippable, OsFam, undefined). + +os_base_skip(Skippable, OsFam, OsName) -> + %% Check if the entire family is to be skipped + %% Example: [win32, unix] case lists:member(OsFam, Skippable) of true -> true; false -> - case lists:keysearch(OsFam, 1, Skippable) of - {value, {OsFam, OsName}} -> - true; - {value, {OsFam, OsNames}} when is_list(OsNames) -> + %% Example: [{unix, freebsd}] | [{unix, [freebsd, darwin]}] + case lists:keysearch(OsFam, 1, Skippable) of + {value, {OsFam, OsName}} -> + true; + {value, {OsFam, OsNames}} when is_list(OsNames) -> + %% OsNames is a list of: + %% [atom()|{atom(), function/0 | function/1}] case lists:member(OsName, OsNames) of true -> true; false -> - case lists:keymember(OsName, 1, OsNames) of - {value, {OsName, Check}} when is_function(Check) -> - Check(); - _ -> - false - end + os_based_skip_check(OsName, OsNames) end; - _ -> - false - end - end; -os_based_skip(_Crap) -> - io:format("os_based_skip -> entry with" - "~n _Crap: ~p" - "~n", [_Crap]), - false. + _ -> + false + end + end. + +%% Performs a check via a provided fun with arity 0 or 1. +%% The argument is the result of os:version(). +os_based_skip_check(OsName, OsNames) -> + case lists:keysearch(OsName, 1, OsNames) of + {value, {OsName, Check}} when is_function(Check, 0) -> + Check(); + {value, {OsName, Check}} when is_function(Check, 1) -> + Check(os:version()); + _ -> + false + end. + + +%% A basic test to check if current host supports IPv6 +has_support_ipv6() -> + case inet:gethostname() of + {ok, Hostname} -> + has_support_ipv6(Hostname); + _ -> + false + end. + +has_support_ipv6(Hostname) -> + case inet:getaddr(Hostname, inet6) of + {ok, Addr} when (size(Addr) =:= 8) andalso + (element(1, Addr) =/= 0) andalso + (element(1, Addr) =/= 16#fe80) -> + true; + {ok, _} -> + false; + {error, _} -> + false + end. + + +is_ipv6_host() -> + case inet:gethostname() of + {ok, Hostname} -> + is_ipv6_host(Hostname); + {error, _} -> + false + end. + +is_ipv6_host(Hostname) -> + case ct:require(ipv6_hosts) of + ok -> + lists:member(list_to_atom(Hostname), ct:get_config(ipv6_hosts)); + _ -> + false + end. %% ---------------------------------------------------------------- diff --git a/lib/snmp/test/snmp_test_lib.hrl b/lib/snmp/test/snmp_test_lib.hrl index 335f3fff3c..c66602b779 100644 --- a/lib/snmp/test/snmp_test_lib.hrl +++ b/lib/snmp/test/snmp_test_lib.hrl @@ -49,8 +49,12 @@ snmp_test_lib:os_based_skip(Skippable)). -define(NON_PC_TC_MAYBE_SKIP(Config, Condition), snmp_test_lib:non_pc_tc_maybe_skip(Config, Condition, ?MODULE, ?LINE)). --define(SKIP(Reason), snmp_test_lib:skip(Reason, ?MODULE, ?LINE)). --define(FAIL(Reason), snmp_test_lib:fail(Reason, ?MODULE, ?LINE)). +-define(SKIP(Reason), snmp_test_lib:skip(Reason, ?MODULE, ?LINE)). +-define(FAIL(Reason), snmp_test_lib:fail(Reason, ?MODULE, ?LINE)). +-define(IS_IPV6_HOST(), snmp_test_lib:is_ipv6_host()). +-define(IS_IPV6_HOST(H), snmp_test_lib:is_ipv6_host(H)). +-define(HAS_SUPPORT_IPV6(), snmp_test_lib:has_support_ipv6()). +-define(HAS_SUPPORT_IPV6(H), snmp_test_lib:has_support_ipv6(H)). %% - Time macros - @@ -150,8 +154,10 @@ snmp_test_lib:print(P, ?MODULE, ?LINE, F, A)). -define(PRINT1(F, A), snmp_test_lib:print1(F, A)). +-define(PRINT1(F), ?PRINT1(F, [])). -define(EPRINT1(F, A), ?PRINT1("<ERROR> " ++ F, A)). -define(PRINT2(F, A), snmp_test_lib:print2(F, A)). +-define(PRINT2(F), ?PRINT2(F, [])). -define(EPRINT2(F, A), ?PRINT2("<ERROR> " ++ F, A)). diff --git a/lib/snmp/test/snmp_test_mgr.erl b/lib/snmp/test/snmp_test_mgr.erl index 9190c07e6d..9d6be65088 100644 --- a/lib/snmp/test/snmp_test_mgr.erl +++ b/lib/snmp/test/snmp_test_mgr.erl @@ -247,10 +247,21 @@ init({Options, CallerPid}) -> IpFamily = get_value(ipfamily, Options, inet), print("[~w] ~p -> IpFamily: ~p~n", [?MODULE, self(), IpFamily]), AgIp = case snmp_misc:assq(agent, Options) of - {value, Tuple4} when is_tuple(Tuple4) andalso - (size(Tuple4) =:= 4) -> - Tuple4; + {value, Addr} when is_tuple(Addr) andalso + (size(Addr) =:= 4) andalso + (IpFamily =:= inet) -> + print("[~w] ~p -> Addr: ~p~n", + [?MODULE, self(), Addr]), + Addr; + {value, Addr} when is_tuple(Addr) andalso + (size(Addr) =:= 8) andalso + (IpFamily =:= inet6) -> + print("[~w] ~p -> Addr: ~p~n", + [?MODULE, self(), Addr]), + Addr; {value, Host} when is_list(Host) -> + print("[~w] ~p -> Host: ~p~n", + [?MODULE, self(), Host]), {ok, Ip} = snmp_misc:ip(Host, IpFamily), Ip end, diff --git a/lib/snmp/test/snmp_test_mgr_misc.erl b/lib/snmp/test/snmp_test_mgr_misc.erl index 315e3ebd9e..6608a88c00 100644 --- a/lib/snmp/test/snmp_test_mgr_misc.erl +++ b/lib/snmp/test/snmp_test_mgr_misc.erl @@ -576,6 +576,7 @@ vsn('version-2') -> v2c. udp_send(UdpId, AgentIp, UdpPort, B) -> + ?vlog("attempt send message (~w bytes) to ~p", [sz(B), {AgentIp, UdpPort}]), case (catch gen_udp:send(UdpId, AgentIp, UdpPort, B)) of {error,ErrorReason} -> error("failed (error) sending message to ~p:~p: " @@ -880,7 +881,7 @@ display_prop_hdr(S) -> %%---------------------------------------------------------------------- sz(L) when is_list(L) -> - length(lists:flatten(L)); + iolist_size(L); sz(B) when is_binary(B) -> size(B); sz(O) -> diff --git a/lib/snmp/test/snmp_to_snmpnet_SUITE.erl b/lib/snmp/test/snmp_to_snmpnet_SUITE.erl index b83c7461d1..b8bdb30271 100644 --- a/lib/snmp/test/snmp_to_snmpnet_SUITE.erl +++ b/lib/snmp/test/snmp_to_snmpnet_SUITE.erl @@ -247,9 +247,17 @@ erlang_agent_netsnmp_get(Config) when is_list(Config) -> start_agent(Config), Oid = ?sysDescr_instance, Expected = expected(Oid, get), - [Expected = snmpget(Oid, Transport, Config) - || Transport <- Transports], - ok. + try + begin + [Expected = snmpget(Oid, Transport, Config) + || Transport <- Transports], + ok + end + catch + throw:{skip, _} = SKIP -> + SKIP + end. + %%-------------------------------------------------------------------- erlang_manager_netsnmp_get() -> @@ -260,29 +268,34 @@ erlang_manager_netsnmp_get(Config) when is_list(Config) -> SysDescr = "Net-SNMP agent", TargetName = "Target Net-SNMP agent", Transports = ?config(transports, Config), - ProgHandle = start_snmpd(Community, SysDescr, Config), - start_manager(Config), - snmp_manager_user:start_link(self(), test_user), - [snmp_manager_user:register_agent( - TargetName++domain_suffix(Domain), - [{reg_type, target_name}, - {tdomain, Domain}, {taddress, Addr}, - {community, Community}, {engine_id, "EngineId"}, - {version, v2}, {sec_model, v2c}, {sec_level, noAuthNoPriv}]) - || {Domain, Addr} <- Transports], - Results = - [snmp_manager_user:sync_get( - TargetName++domain_suffix(Domain), - [?sysDescr_instance]) - || {Domain, _} <- Transports], - ct:pal("sync_get -> ~p", [Results]), - snmp_manager_user:stop(), - stop_program(ProgHandle), - [{ok, - {noError, 0, - [{varbind, ?sysDescr_instance, 'OCTET STRING', SysDescr,1}] }, - _} = R || R <- Results], - ok. + case start_snmpd(Community, SysDescr, Config) of + {skip, _} = SKIP -> + SKIP; + ProgHandle -> + start_manager(Config), + snmp_manager_user:start_link(self(), test_user), + [snmp_manager_user:register_agent( + TargetName++domain_suffix(Domain), + [{reg_type, target_name}, + {tdomain, Domain}, {taddress, Addr}, + {community, Community}, {engine_id, "EngineId"}, + {version, v2}, {sec_model, v2c}, {sec_level, noAuthNoPriv}]) + || {Domain, Addr} <- Transports], + Results = + [snmp_manager_user:sync_get( + TargetName++domain_suffix(Domain), + [?sysDescr_instance]) + || {Domain, _} <- Transports], + ct:pal("sync_get -> ~p", [Results]), + snmp_manager_user:stop(), + stop_program(ProgHandle), + [{ok, + {noError, 0, + [{varbind, ?sysDescr_instance, 'OCTET STRING', SysDescr,1}] }, + _} = R || R <- Results], + ok + end. + %%-------------------------------------------------------------------- erlang_agent_netsnmp_inform(Config) when is_list(Config) -> @@ -292,17 +305,19 @@ erlang_agent_netsnmp_inform(Config) when is_list(Config) -> start_agent(Config), ok = snmpa:load_mib(snmp_master_agent, filename:join(DataDir, Mib)), - ProgHandle = start_snmptrapd(Mib, Config), - - snmpa:send_notification( - snmp_master_agent, testTrapv22, {erlang_agent_test, self()}), - - receive - {snmp_targets, erlang_agent_test, Addresses} -> - ct:pal("Notification sent to: ~p~n", [Addresses]), - erlang_agent_netsnmp_inform_responses(Addresses) - end, - stop_program(ProgHandle). + case start_snmptrapd(Mib, Config) of + {skip, _} = SKIP -> + SKIP; + ProgHandle -> + snmpa:send_notification( + snmp_master_agent, testTrapv22, {erlang_agent_test, self()}), + receive + {snmp_targets, erlang_agent_test, Addresses} -> + ct:pal("Notification sent to: ~p~n", [Addresses]), + erlang_agent_netsnmp_inform_responses(Addresses) + end, + stop_program(ProgHandle) + end. erlang_agent_netsnmp_inform_responses([]) -> receive @@ -326,6 +341,7 @@ erlang_agent_netsnmp_inform_responses([Address | Addresses]) -> %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- + snmpget(Oid, Transport, Config) -> Versions = ?config(snmp_versions, Config), @@ -335,10 +351,14 @@ snmpget(Oid, Transport, Config) -> "-Cf", net_snmp_addr_str(Transport), oid_str(Oid)], - ProgHandle = start_program(snmpget, Args, none, Config), - {_, line, Line} = get_program_output(ProgHandle), - stop_program(ProgHandle), - Line. + case start_program(snmpget, Args, none, Config) of + {skip, _} = SKIP -> + throw(SKIP); + ProgHandle -> + {_, line, Line} = get_program_output(ProgHandle), + stop_program(ProgHandle), + Line + end. start_snmptrapd(Mibs, Config) -> DataDir = ?config(data_dir, Config), @@ -382,12 +402,13 @@ start_program(Prog, Args, StartCheckMP, Config) -> DataDir = ?config(data_dir, Config), StartWrapper = filename:join(DataDir, "start_stop_wrapper"), Parent = self(), - Pid = - spawn_link( + %% process_flag(trap_exit, true), + {Pid, Mon} = + spawn_monitor( fun () -> run_program(Parent, StartWrapper, [Path | Args]) end), - start_check(Pid, erlang:monitor(process, Pid), StartCheckMP). + start_check(Pid, Mon, StartCheckMP). start_check(Pid, Mon, none) -> {Pid, Mon}; @@ -400,6 +421,10 @@ start_check(Pid, Mon, StartCheckMP) -> nomatch -> start_check(Pid, Mon, StartCheckMP) end; + {'DOWN', Mon, _, _Pid, {skip, Reason} = SKIP} -> + ct:pal("Received DOWN from ~p" + "~n Skip Reason: ~p", [_Pid, Reason]), + SKIP; {'DOWN', Mon, _, _, Reason} -> ct:fail("Prog ~p start failed: ~p", [Pid, Reason]) end. @@ -446,14 +471,34 @@ run_program_loop(Parent, Port, Buf) -> {Port, {data, {Flag, Data}}} -> case Flag of eol -> - Line = iolist_to_binary(lists:reverse(Buf, Data)), - ct:pal("Prog ~p output: ~s", [Port, Line]), - Parent ! {self(), line, Line}, - run_program_loop(Parent, Port, []); + Line = iolist_to_binary(lists:reverse(Buf, Data)), + ct:pal("Prog ~p output: ~s", [Port, Line]), + %% There are potentially many different fail outputs, + %% but for now we test for just this one: illegal option + IOpt = "illegal option", + case string:find(binary_to_list(Line), IOpt) of + nomatch -> + Parent ! {self(), line, Line}, + run_program_loop(Parent, Port, []); + Line2 -> + %% Try to extract the actual illegal option string + IOpt2 = + case string:take( + string:prefix(Line2, IOpt), [$-, $ ]) of + {_, Str} when length(Str) > 0 -> + Str; + _X -> + Line2 + end, + ct:pal("Force program ~p stop", [Port]), + true = port_command(Port, <<"stop\n">>), + (catch port_close(Port)), + exit({skip, {illegal_option, IOpt2}}) + end; noeol -> run_program_loop(Parent, Port, [Data | Buf]) end; - {Port, {exit_status,ExitStatus}} -> + {Port, {exit_status, ExitStatus}} -> ct:pal("Prog ~p exit: ~p", [Port, ExitStatus]), catch port_close(Port), Parent ! {self(), exit, ExitStatus}; diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 3fd6eae423..8b7cb4dcd4 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -347,6 +347,7 @@ <datatype> <name name="subsystem_daemon_option"/> + <name name="subsystem_specs"/> <name name="subsystem_spec"/> <desc> <p>Defines a subsystem in the daemon.</p> diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 54e98ee10e..0014002192 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -312,7 +312,8 @@ | gen_tcp:listen_option() | ?COMMON_OPTION . --type subsystem_daemon_option() :: {subsystems, subsystem_spec()}. +-type subsystem_daemon_option() :: {subsystems, subsystem_specs()}. +-type subsystem_specs() :: [ subsystem_spec() ]. -type shell_daemon_option() :: {shell, mod_fun_args() | 'shell_fun/1'() | 'shell_fun/2'() }. -type 'shell_fun/1'() :: fun((User::string()) -> pid()) . diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 5dde0f504e..f320b4c006 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,36 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 9.3.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Correct handshake handling, might cause strange symptoms + such as ASN.1 certificate decoding issues.</p> + <p> + Own Id: OTP-15879 Aux Id: ERL-968 </p> + </item> + <item> + <p> + Fix handling of the signature_algorithms_cert extension + in the ClientHello handshake message.</p> + <p> + Own Id: OTP-15887 Aux Id: ERL-973 </p> + </item> + <item> + <p> + Handle new ClientHello extensions when handshake is + paused by the {handshake, hello} ssl option.</p> + <p> + Own Id: OTP-15888 Aux Id: ERL-975 </p> + </item> + </list> + </section> + +</section> + <section><title>SSL 9.3.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index dabc2f8ec8..323d9e3284 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -259,7 +259,7 @@ next_event(StateName, Record, State) -> next_event(StateName, no_record, State0, Actions) -> case next_record(StateName, State0) of {no_record, State} -> - {next_state, StateName, State, Actions}; + ssl_connection:hibernate_after(StateName, State, Actions); {Record, State} -> next_event(StateName, Record, State, Actions) end; diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 4de51c9a35..a0ae51ed0a 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -1180,7 +1180,7 @@ validate_certificate_chain(Certs, CertDbHandle, CertDbRef, SslOptions, CRLDbHand CertDbHandle, CertDbRef)} end catch - error:{badmatch,{asn1, Asn1Reason}} -> + error:{badmatch,{error, {asn1, Asn1Reason}}} -> %% ASN-1 decode of certificate somehow failed {error, {certificate_unknown, {failed_to_decode_certificate, Asn1Reason}}}; error:OtherReason -> diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index a5aa81a67d..0cdfea0ac2 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -3676,7 +3676,7 @@ hibernate(Config) -> ssl_test_lib:check_result(Server, ok, Client, ok), - timer:sleep(1500), + ct:sleep(1500), {current_function, {erlang, hibernate, 3}} = process_info(Pid, current_function), @@ -3712,6 +3712,8 @@ hibernate_right_away(Config) -> [{port, Port1}, {options, [{hibernate_after, 0}|ClientOpts]}]), ssl_test_lib:check_result(Server1, ok, Client1, ok), + + ct:sleep(1000), %% Schedule out {current_function, {erlang, hibernate, 3}} = process_info(Pid1, current_function), diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 4e387fa403..01dee392f5 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 9.3.2 +SSL_VSN = 9.3.3 diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 726b409d4d..197564b895 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -33,6 +33,8 @@ %%% BIFs +-export([internal_run/4]). + -export([version/0, compile/1, compile/2, run/2, run/3, inspect/2]). -spec version() -> binary(). @@ -100,6 +102,40 @@ run(_, _) -> run(_, _, _) -> erlang:nif_error(undef). +-spec internal_run(Subject, RE, Options, FirstCall) -> {match, Captured} | + match | + nomatch | + {error, ErrType} when + Subject :: iodata() | unicode:charlist(), + RE :: mp() | iodata() | unicode:charlist(), + Options :: [Option], + Option :: anchored | global | notbol | noteol | notempty + | notempty_atstart | report_errors + | {offset, non_neg_integer()} | + {match_limit, non_neg_integer()} | + {match_limit_recursion, non_neg_integer()} | + {newline, NLSpec :: nl_spec()} | + bsr_anycrlf | bsr_unicode | {capture, ValueSpec} | + {capture, ValueSpec, Type} | CompileOpt, + Type :: index | list | binary, + ValueSpec :: all | all_but_first | all_names | first | none | ValueList, + ValueList :: [ValueID], + ValueID :: integer() | string() | atom(), + CompileOpt :: compile_option(), + Captured :: [CaptureData] | [[CaptureData]], + CaptureData :: {integer(), integer()} + | ListConversionData + | binary(), + ListConversionData :: string() + | {error, string(), binary()} + | {incomplete, string(), binary()}, + ErrType :: match_limit | match_limit_recursion | {compile, CompileErr}, + CompileErr :: {ErrString :: string(), Position :: non_neg_integer()}, + FirstCall :: boolean(). + +internal_run(_, _, _, _) -> + erlang:nif_error(undef). + -spec inspect(MP,Item) -> {namelist, [ binary() ]} when MP :: mp(), Item :: namelist. @@ -765,17 +801,17 @@ do_grun(FlatSubject,Subject,Unicode,CRLF,RE,{Options0,NeedClean}) -> try postprocess(loopexec(FlatSubject,RE,InitialOffset, byte_size(FlatSubject), - Unicode,CRLF,StrippedOptions), + Unicode,CRLF,StrippedOptions,true), SelectReturn,ConvertReturn,FlatSubject,Unicode) catch throw:ErrTuple -> ErrTuple end. -loopexec(_,_,X,Y,_,_,_) when X > Y -> +loopexec(_,_,X,Y,_,_,_,_) when X > Y -> {match,[]}; -loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> - case re:run(Subject,RE,[{offset,X}]++Options) of +loopexec(Subject,RE,X,Y,Unicode,CRLF,Options, First) -> + case re:internal_run(Subject,RE,[{offset,X}]++Options,First) of {error, Err} -> throw({error,Err}); nomatch -> @@ -784,11 +820,11 @@ loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> {match,Rest} = case B>0 of true -> - loopexec(Subject,RE,A+B,Y,Unicode,CRLF,Options); + loopexec(Subject,RE,A+B,Y,Unicode,CRLF,Options,false); false -> {match,M} = - case re:run(Subject,RE,[{offset,X},notempty_atstart, - anchored]++Options) of + case re:internal_run(Subject,RE,[{offset,X},notempty_atstart, + anchored]++Options,false) of nomatch -> {match,[]}; {match,Other} -> @@ -801,7 +837,7 @@ loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> forward(Subject,A,1,Unicode,CRLF) end, {match,MM} = loopexec(Subject,RE,NewA,Y, - Unicode,CRLF,Options), + Unicode,CRLF,Options,false), case M of [] -> {match,MM}; diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index ecb514e9f3..d7d57941c2 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -108,7 +108,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-10.4","crypto-3.3", + {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-@OTP-15831:OTP-15836@","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index dd49288417..09238ae2b4 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -75,7 +75,8 @@ -export([throughput_benchmark/0, throughput_benchmark/1, test_throughput_benchmark/1, - long_throughput_benchmark/1]). + long_throughput_benchmark/1, + lookup_catree_par_vs_seq_init_benchmark/0]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -6728,6 +6729,14 @@ do_work(WorksDoneSoFar, Table, ProbHelpTab, Range, Operations) -> end. prefill_table(T, KeyRange, Num, ObjFun) -> + Parent = self(), + spawn_link(fun() -> + prefill_table_helper(T, KeyRange, Num, ObjFun), + Parent ! done + end), + receive done -> ok end. + +prefill_table_helper(T, KeyRange, Num, ObjFun) -> Seed = rand:uniform(KeyRange), %%io:format("prefill_table: Seed = ~p\n", [Seed]), RState = unique_rand_start(KeyRange, Seed), @@ -6740,11 +6749,77 @@ prefill_table_loop(T, RS0, N, ObjFun) -> ets:insert(T, ObjFun(Key)), prefill_table_loop(T, RS1, N-1, ObjFun). +inserter_proc_starter(T, ToInsert, Parent) -> + receive + start -> ok + end, + inserter_proc(T, ToInsert, [], Parent, false). + +inserter_proc(T, [], Inserted, Parent, _) -> + inserter_proc(T, Inserted, [], Parent, true); +inserter_proc(T, [I | ToInsert], Inserted, Parent, CanStop) -> + Stop = + case CanStop of + true -> + receive + stop -> Parent ! stopped + after 0 -> no_stop + end; + false -> no_stop + end, + case Stop of + no_stop -> + ets:insert(T, I), + inserter_proc(T, ToInsert, [I | Inserted], Parent, CanStop); + _ -> ok + end. + +prefill_table_parallel(T, KeyRange, Num, ObjFun) -> + Parent = self(), + spawn_link(fun() -> + prefill_table_parallel_helper(T, KeyRange, Num, ObjFun), + Parent ! done + end), + receive done -> ok end. + +prefill_table_parallel_helper(T, KeyRange, Num, ObjFun) -> + NrOfSchedulers = erlang:system_info(schedulers), + Seed = rand:uniform(KeyRange), + %%io:format("prefill_table: Seed = ~p\n", [Seed]), + RState = unique_rand_start(KeyRange, Seed), + InsertMap = prefill_insert_map_loop(T, RState, Num, ObjFun, #{}, NrOfSchedulers), + Self = self(), + Pids = [ + begin + InserterFun = + fun() -> + inserter_proc_starter(T, ToInsert, Self) + end, + spawn_link(InserterFun) + end + || ToInsert <- maps:values(InsertMap)], + [Pid ! start || Pid <- Pids], + timer:sleep(1000), + [Pid ! stop || Pid <- Pids], + [receive stopped -> ok end || _Pid <- Pids]. + +prefill_insert_map_loop(_, _, 0, _, InsertMap, _NrOfSchedulers) -> + InsertMap; +prefill_insert_map_loop(T, RS0, N, ObjFun, InsertMap, NrOfSchedulers) -> + {Key, RS1} = unique_rand_next(RS0), + Sched = N rem NrOfSchedulers, + PrevInserts = maps:get(Sched, InsertMap, []), + NewPrevInserts = [ObjFun(Key) | PrevInserts], + NewInsertMap = maps:put(Sched, NewPrevInserts, InsertMap), + prefill_insert_map_loop(T, RS1, N-1, ObjFun, NewInsertMap, NrOfSchedulers). + -record(ets_throughput_bench_config, {benchmark_duration_ms = 3000, recover_time_ms = 1000, thread_counts = not_set, key_ranges = [1000000], + init_functions = [fun prefill_table/4], + nr_of_repeats = 1, scenarios = [ [ @@ -6838,7 +6913,7 @@ prefill_table_loop(T, RS0, N, ObjFun) -> notify_res_fun = fun(_Name, _Throughput) -> ok end, print_result_paths_fun = fun(ResultPath, _LatestResultPath) -> - Comment = + Comment = io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]), {comment, Comment} end @@ -6848,7 +6923,7 @@ stdout_notify_res(ResultPath, LatestResultPath) -> io:format("Result Location: /~s~n", [ResultPath]), io:format("Latest Result Location: ~s~n", [LatestResultPath]). -throughput_benchmark() -> +throughput_benchmark() -> throughput_benchmark( #ets_throughput_bench_config{ print_result_paths_fun = fun stdout_notify_res/2}). @@ -6856,9 +6931,11 @@ throughput_benchmark() -> throughput_benchmark( #ets_throughput_bench_config{ benchmark_duration_ms = BenchmarkDurationMs, - recover_time_ms = RecoverTimeMs, - thread_counts = ThreadCountsOpt, - key_ranges = KeyRanges, + recover_time_ms = RecoverTimeMs, + thread_counts = ThreadCountsOpt, + key_ranges = KeyRanges, + init_functions = InitFuns, + nr_of_repeats = NrOfRepeats, scenarios = Scenarios, table_types = TableTypes, etsmem_fun = ETSMemFun, @@ -6872,21 +6949,21 @@ throughput_benchmark( Start = rand:uniform(KeyRange), Last = lists:foldl( - fun(_, Prev) -> + fun(_, Prev) -> case Prev of '$end_of_table'-> ok; _ -> try ets:next(T, Prev) of Normal -> Normal catch - error:badarg -> + error:badarg -> % sets (not ordered_sets) cannot handle when the argument % to next is not in the set rand:uniform(KeyRange) end end end, - Start, + Start, lists:seq(1, SeqSize)), case Last =:= -1 of true -> io:format("Will never be printed"); @@ -6898,26 +6975,26 @@ throughput_benchmark( Start = rand:uniform(KeyRange), Last = Start + SeqSize, case -1 =:= ets:select_count(T, - ets:fun2ms(fun({X}) when X > Start andalso X =< Last -> true end)) of + ets:fun2ms(fun({X}) when X > Start andalso X =< Last -> true end)) of true -> io:format("Will never be printed"); false -> ok end end, %% Mapping benchmark operation names to their corresponding functions that do them - Operations = + Operations = #{insert => - fun(T,KeyRange) -> + fun(T,KeyRange) -> Num = rand:uniform(KeyRange), ets:insert(T, {Num}) end, delete => - fun(T,KeyRange) -> + fun(T,KeyRange) -> Num = rand:uniform(KeyRange), ets:delete(T, Num) end, lookup => - fun(T,KeyRange) -> + fun(T,KeyRange) -> Num = rand:uniform(KeyRange), ets:lookup(T, Num) end, @@ -6928,8 +7005,8 @@ throughput_benchmark( nextseq1000 => fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, selectAll => - fun(T,_KeyRange) -> - case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of + fun(T,_KeyRange) -> + case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of true -> io:format("Will never be printed"); false -> ok end @@ -6951,7 +7028,7 @@ throughput_benchmark( NewCurrent = Current + OpPropability, [{NewCurrent, OpName}| Calculate(Res, NewCurrent)] end, - RenderScenario = + RenderScenario = fun R([], StringSoFar) -> StringSoFar; R([{Fraction, Operation}], StringSoFar) -> @@ -6978,7 +7055,7 @@ throughput_benchmark( false -> ok end end, - DataHolder = + DataHolder = fun DataHolderFun(Data)-> receive {get_data, Pid} -> Pid ! {ets_bench_data, Data}; @@ -6992,18 +7069,21 @@ throughput_benchmark( DataHolderPid ! io_lib:format(Str, List) end, GetData = - fun () -> + fun () -> DataHolderPid ! {get_data, self()}, receive {ets_bench_data, Data} -> Data end end, %% Function that runs a benchmark instance and returns the number %% of operations that were performed RunBenchmark = - fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) -> + fun({NrOfProcs, TableConfig, Scenario, Range, Duration, InitFun}) -> ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), Table = ets:new(t, TableConfig), Nobj = Range div 2, - prefill_table(Table, Range, Nobj, fun(K) -> {K} end), + case InitFun of + not_set -> prefill_table(Table, Range, Nobj, fun(K) -> {K} end); + _ -> InitFun(Table, Range, Nobj, fun(K) -> {K} end) + end, Nobj = ets:info(Table, size), SafeFixTableIfRequired(Table, Scenario, true), ParentPid = self(), @@ -7016,12 +7096,14 @@ throughput_benchmark( end, ChildPids = lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)), + erlang:garbage_collect(), + timer:sleep(RecoverTimeMs), lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), timer:sleep(Duration), lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), TotalWorksDone = lists:foldl( - fun(_, Sum) -> - receive + fun(_, Sum) -> + receive Count -> Sum + Count end end, 0, ChildPids), @@ -7032,27 +7114,32 @@ throughput_benchmark( RunBenchmarkInSepProcess = fun(ParameterTuple) -> P = self(), - spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), - Result = receive {bench_result, Res} -> Res end, - timer:sleep(RecoverTimeMs), - Result + Results = + [begin + spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), + receive {bench_result, Res} -> Res end + end || _ <- lists:seq(1, NrOfRepeats)], + lists:sum(Results) / NrOfRepeats end, RunBenchmarkAndReport = fun(ThreadCount, TableType, Scenario, KeyRange, - Duration) -> + Duration, + InitFunName, + InitFun) -> Result = RunBenchmarkInSepProcess({ThreadCount, TableType, Scenario, KeyRange, - Duration}), + Duration, + InitFun}), Throughput = Result/(Duration/1000.0), PrintData("; ~f",[Throughput]), - Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, " + Name = io_lib:format("Scenario: ~s, ~w, Key Range Size: ~w, " "# of Processes: ~w, Table Type: ~w", - [Scenario, KeyRange, ThreadCount, TableType]), + [InitFunName, Scenario, KeyRange, ThreadCount, TableType]), NotifyResFun(Name, Throughput) end, ThreadCounts = @@ -7087,17 +7174,29 @@ throughput_benchmark( PrintData("$~n",[]), lists:foreach( fun(TableType) -> - PrintData("~w ",[TableType]), lists:foreach( - fun(ThreadCount) -> - RunBenchmarkAndReport(ThreadCount, - TableType, - Scenario, - KeyRange, - BenchmarkDurationMs) + fun(InitFunArg) -> + {InitFunName, InitFun} = + case InitFunArg of + {FunName, Fun} -> {FunName, Fun}; + Fun -> {"", Fun} + end, + PrintData("~s,~w ",[InitFunName,TableType]), + lists:foreach( + fun(ThreadCount) -> + RunBenchmarkAndReport(ThreadCount, + TableType, + Scenario, + KeyRange, + BenchmarkDurationMs, + InitFunName, + InitFun) + end, + ThreadCounts), + PrintData("$~n",[]) end, - ThreadCounts), - PrintData("$~n",[]) + InitFuns) + end, TableTypes) end, @@ -7121,7 +7220,7 @@ throughput_benchmark( test_throughput_benchmark(Config) when is_list(Config) -> throughput_benchmark( #ets_throughput_bench_config{ - benchmark_duration_ms = 100, + benchmark_duration_ms = 100, recover_time_ms = 0, thread_counts = [1, erlang:system_info(schedulers)], key_ranges = [50000], @@ -7136,7 +7235,7 @@ long_throughput_benchmark(Config) when is_list(Config) -> recover_time_ms = 1000, thread_counts = [1, N div 2, N], key_ranges = [1000000], - scenarios = + scenarios = [ [ {0.5, insert}, @@ -7171,15 +7270,15 @@ long_throughput_benchmark(Config) when is_list(Config) -> {0.01, partial_select1000} ] ], - table_types = + table_types = [ [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], [set, public, {write_concurrency, true}, {read_concurrency, true}] ], etsmem_fun = fun etsmem/0, verify_etsmem_fun = fun verify_etsmem/1, - notify_res_fun = - fun(Name, Throughput) -> + notify_res_fun = + fun(Name, Throughput) -> SummaryTable = proplists:get_value(ets_benchmark_result_summary_tab, Config), AddToSummaryCounter = @@ -7209,13 +7308,47 @@ long_throughput_benchmark(Config) when is_list(Config) -> total_throughput_ordered_set) end, ct_event:notify( - #event{name = benchmark_data, + #event{name = benchmark_data, data = [{suite,"ets_bench"}, {name, Name}, {value,Throughput}]}) end }). +%% This function compares the lookup operation's performance for +%% ordered_set ETS tables with and without write_concurrency enabled +%% when the data structures have been populated in parallel and +%% sequentially. +%% +%% The main purpose of this function is to check that the +%% implementation of ordered_set with write_concurrency (CA tree) +%% adapts its structure to contention even when only lookup operations +%% are used. +lookup_catree_par_vs_seq_init_benchmark() -> + N = erlang:system_info(schedulers), + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 600000, + recover_time_ms = 1000, + thread_counts = [1, N div 2, N], + key_ranges = [1000000], + init_functions = [{"seq_init", fun prefill_table/4}, + {"par_init", fun prefill_table_parallel/4}], + nr_of_repeats = 1, + scenarios = + [ + [ + {1.0, lookup} + ] + ], + table_types = + [ + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public] + ], + print_result_paths_fun = fun stdout_notify_res/2 + }). + add_lists(L1,L2) -> add_lists(L1,L2,[]). add_lists([],[],Acc) -> diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index c9ef9da990..06d8fe9255 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -28,7 +28,8 @@ pcre_compile_workspace_overflow/1,re_infinite_loop/1, re_backwards_accented/1,opt_dupnames/1,opt_all_names/1,inspect/1, opt_no_start_optimize/1,opt_never_utf/1,opt_ucp/1, - match_limit/1,sub_binaries/1,copt/1]). + match_limit/1,sub_binaries/1,copt/1,global_unicode_validation/1, + yield_on_subject_validation/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -45,7 +46,8 @@ all() -> pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, inspect, opt_no_start_optimize,opt_never_utf,opt_ucp, - match_limit, sub_binaries, re_version]. + match_limit, sub_binaries, re_version, global_unicode_validation, + yield_on_subject_validation]. groups() -> []. @@ -200,7 +202,58 @@ re_version(_Config) -> {match,[Version]} = re:run(Version,"^[0-9]\\.[0-9]{2} 20[0-9]{2}-[0-9]{2}-[0-9]{2}",[{capture,all,binary}]), ok. +global_unicode_validation(Config) when is_list(Config) -> + %% Test that unicode validation of the subject is not done + %% for every match found... + Bin = binary:copy(<<"abc\n">>,100000), + {TimeAscii, _} = take_time(fun () -> + re:run(Bin, <<"b">>, [global]) + end), + {TimeUnicode, _} = take_time(fun () -> + re:run(Bin, <<"b">>, [unicode,global]) + end), + if TimeAscii == 0; TimeUnicode == 0 -> + {comment, "Not good enough resolution to compare results"}; + true -> + %% The time the operations takes should be in the + %% same order of magnitude. If validation of the + %% whole subject occurs for every match, the unicode + %% variant will take way longer time... + true = TimeUnicode div TimeAscii < 10 + end. + +take_time(Fun) -> + Start = erlang:monotonic_time(nanosecond), + Res = Fun(), + End = erlang:monotonic_time(nanosecond), + {End-Start, Res}. + +yield_on_subject_validation(Config) when is_list(Config) -> + Go = make_ref(), + Bin = binary:copy(<<"abc\n">>,100000), + {P, M} = spawn_opt(fun () -> + receive Go -> ok end, + {match,[{1,1}]} = re:run(Bin, <<"b">>, [unicode]) + end, + [link, monitor]), + 1 = erlang:trace(P, true, [running]), + P ! Go, + N = count_re_run_trap_out(P, M), + true = N >= 5, + ok. +count_re_run_trap_out(P, M) when is_reference(M) -> + receive {'DOWN',M,process,P,normal} -> ok end, + TD = erlang:trace_delivered(P), + receive {trace_delivered, P, TD} -> ok end, + count_re_run_trap_out(P, 0); +count_re_run_trap_out(P, N) when is_integer(N) -> + receive + {trace,P,out,{erlang,re_run_trap,3}} -> + count_re_run_trap_out(P, N+1) + after 0 -> + N + end. %% Test compile options given directly to run. combined_options(Config) when is_list(Config) -> diff --git a/lib/wx/c_src/Makefile.in b/lib/wx/c_src/Makefile.in index 8ec64bea7e..daadf5e785 100644 --- a/lib/wx/c_src/Makefile.in +++ b/lib/wx/c_src/Makefile.in @@ -70,7 +70,7 @@ else RC_FILE = endif -WX_OBJECTS = $(GENERAL_O) $(GENERATED_O) $(RC_FILE) +WX_OBJECTS = $(GENERATED_O) $(GENERAL_O) $(RC_FILE) OBJECTS = $(WX_OBJECTS) $(GL_OBJECTS) @@ -86,7 +86,7 @@ LD = $(CXX) LDFLAGS = @LDFLAGS@ RESCOMP = @WX_RESCOMP@ -ifeq (@WX_HAVE_STATIC_LIBS@,true) +ifeq (@WX_HAVE_STATIC_LIBS@,true) OPT_WX_LIBS = @WX_LIBS_STATIC@ DEBUG_WX_LIBS = @DEBUG_WX_LIBS_STATIC@ else @@ -97,14 +97,16 @@ endif ifeq ($(TYPE),debug) WX_CFLAGS = @DEBUG_WX_CFLAGS@ CFLAGS = @DEBUG_CFLAGS@ -WX_CXX_FLAGS = @DEBUG_WX_CXXFLAGS@ +WX_CXX_FLAGS = @DEBUG_WX_CXXFLAGS@ CXX_FLAGS = @DEBUG_CXXFLAGS@ +CXX_NO_OPT_FLAGS = @DEBUG_CXXFLAGS@ WX_LIBS = $(DEBUG_WX_LIBS) else WX_CFLAGS = @WX_CFLAGS@ CFLAGS = @CFLAGS@ -WX_CXX_FLAGS = @WX_CXXFLAGS@ +WX_CXX_FLAGS = @WX_CXXFLAGS@ CXX_FLAGS = @CXXFLAGS@ +CXX_NO_OPT_FLAGS = @CXXNOOPTFLAGS@ WX_LIBS = $(OPT_WX_LIBS) endif @@ -113,6 +115,7 @@ GL_LIBS = @GL_LIBS@ CC_O = $(V_CC) -c $(CFLAGS) $(WX_CFLAGS) $(COMMON_CFLAGS) OBJC_CC_O = $(OBJC_CC) -c $(CFLAGS) $(OBJC_CFLAGS) $(WX_CFLAGS) $(COMMON_CFLAGS) CXX_O = $(V_CXX) -c $(CXX_FLAGS) $(WX_CXX_FLAGS) $(COMMON_CFLAGS) +CXX_O_NO_OPT = $(V_CXX) -c $(CXX_NO_OPT_FLAGS) $(WX_CXX_FLAGS) $(COMMON_CFLAGS) # Targets @@ -152,6 +155,10 @@ $(SYS_TYPE)/wxe_ps_init.o: wxe_ps_init.c $(V_at)mkdir -p $(SYS_TYPE) $(cc_verbose)$(OBJC_CC_O) $< -o $@ +$(SYS_TYPE)/wxe_funcs.o: gen/wxe_funcs.cpp + $(V_at)mkdir -p $(SYS_TYPE) + $(CXX_O_NO_OPT) $< -o $@ + $(SYS_TYPE)/%.o: gen/%.cpp $(V_at)mkdir -p $(SYS_TYPE) $(CXX_O) $< -o $@ diff --git a/lib/wx/configure.in b/lib/wx/configure.in index dbe237cd74..f35e6cdbd0 100644 --- a/lib/wx/configure.in +++ b/lib/wx/configure.in @@ -30,17 +30,11 @@ if test -f ./CONF_INFO; then fi if test -z "$ERL_TOP" || test ! -d $ERL_TOP ; then - AC_CONFIG_AUX_DIRS(autoconf) - WX_BUILDING_INSIDE_ERLSRC=false + AC_MSG_ERROR([ERL_TOP is not set]) else erl_top=${ERL_TOP} - if test -d $erl_top/erts/autoconf; then - AC_CONFIG_AUX_DIRS($erl_top/erts/autoconf) - WX_BUILDING_INSIDE_ERLSRC=true - else - AC_CONFIG_AUX_DIRS(autoconf) - WX_BUILDING_INSIDE_ERLSRC=false - fi + AC_CONFIG_AUX_DIRS($erl_top/erts/autoconf) + WX_BUILDING_INSIDE_ERLSRC=true fi if test "X$host" != "Xfree_source" -a "X$host" != "Xwin32"; then @@ -67,6 +61,20 @@ AC_PROG_CXX AC_PROG_RANLIB AC_PROG_CPP +AC_LANG_PUSH([C++]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [[#ifndef __cplusplus + #error "broken C++" + #endif]])],, + [CXX=;]) +AC_LANG_POP([C++]) +if test "X$CXX" = X ; then + echo "Can not find C++ compiler" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Can not find C++ compiler]) +fi +WXERL_CAN_BUILD_DRIVER=false + AC_MSG_NOTICE(Building for [$host_os]) WXERL_CAN_BUILD_DRIVER=true @@ -139,13 +147,9 @@ case $host_os in if test X$APPLE_CC = X -o X$APPLE_CXX = X; then AC_MSG_RESULT([no]) dnl Complete failure, we cannot build Cocoa code - if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_ERROR([Can not find compiler to compile Cocoa applications]) - else - echo "Can not find compiler to compile Cocoa applications" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([Can not find compiler to compile Cocoa applications]) - fi + echo "Can not find compiler to compile Cocoa applications" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Can not find compiler to compile Cocoa applications]) WXERL_CAN_BUILD_DRIVER=false else dnl We think we found an Apple compiler and will add @@ -225,11 +229,21 @@ case $host_os in ;; *) DEBUG_CFLAGS="-g -Wall -fPIC $CFLAGS -DDEBUG" - CFLAGS="-g -Wall -O2 -fPIC $CFLAGS -fomit-frame-pointer -fno-strict-aliasing" + CFLAGS="-Wall -fPIC $CFLAGS -fomit-frame-pointer -fno-strict-aliasing" ;; esac -dnl +dnl +dnl Use -O1 -fno-move-loop-invariants for wxe_funcs.cpp to reduce +dnl compilation time +dnl + +if test "x$GCC" = xyes -a X"$host_os" != X"win32" ; then + CXXNOOPT="-O1" + LM_TRY_ENABLE_CFLAG([-fno-move-loop-invariants], [CXXNOOPT]) +fi + +dnl dnl Opengl tests dnl @@ -251,11 +265,13 @@ if test X"$host_os" != X"win32" ; then AC_CHECK_HEADERS([GL/gl.h]) if test X"$ac_cv_header_GL_gl_h" != Xyes ; then AC_MSG_WARN([No OpenGL headers found, wx will NOT be usable]) + echo "No OpenGL headers found, wx will NOT be usable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false CPPFLAGS="$saved_CPPFLAGS" - else + else GL_LIBS="-L/usr/local/lib $GL_LIBS" fi - else + else GL_LIBS="-L/usr/X11R6/lib $GL_LIBS" fi fi @@ -270,6 +286,8 @@ if test X"$host_os" != X"win32" ; then test X"$ac_cv_header_OpenGL_glu_h" != Xyes then AC_MSG_WARN([No GLU headers found, wx will NOT be usable]) + echo "No GLU headers (glu.h) found, wx will NOT be usable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false fi else AC_CHECK_HEADERS([gl/glu.h],[],[],[#include <windows.h>]) @@ -280,47 +298,17 @@ AC_SUBST(GL_LIBS) DEBUG_CXXFLAGS="$CXXFLAGS $DEBUG_CFLAGS $CPPFLAGS" DEBUG_CFLAGS="$DEBUG_CFLAGS $CPPFLAGS $C_ONLY_FLAGS" -CXXFLAGS="$CXXFLAGS $CFLAGS $CPPFLAGS" +CXXNOOPTFLAGS="$CXXFLAGS $CFLAGS $CPPFLAGS $CXXNOOPT" +CXXFLAGS="$CXXFLAGS $CFLAGS $CPPFLAGS" CFLAGS="$CFLAGS $CPPFLAGS $C_ONLY_FLAGS" AC_SUBST(DEBUG_CFLAGS) AC_SUBST(DEBUG_CXXFLAGS) - -if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_CHECKING(for erl) - if test X$ERL != X; then - AC_MSG_RESULT([yes; using $ERL]) - else - type erl >/dev/null 2>&1 - if test $? -eq 0 ; then - ERL=erl - AC_MSG_RESULT([yes]) - else - AC_MSG_ERROR([Cannot find erl in path]) - fi - fi - AC_MSG_CHECKING(for erlc) - if test X$ERLC != X; then - AC_MSG_RESULT([yes; using $ERLC]) - else - type erlc >/dev/null 2>&1 - if test $? -eq 0 ; then - ERLC=erlc - AC_MSG_RESULT([yes]) - else - AC_MSG_ERROR([Cannot find erlc in path]) - fi - fi - ERLANG_ROOT_DIR=`erl -noshell -eval 'io:format("~s~n",[[code:root_dir()]])' -s erlang halt` - AC_MSG_NOTICE(ERL ROOT DIR: [$ERLANG_ROOT_DIR]) - ERLWX_VSN=`grep WX_VSN $srcdir/vsn.mk | sed 's/^.*=[ ]*//'` -else - ERLC=erlc - ERL=erl - ERLANG_ROOT_DIR=$ERL_TOP - AC_SUBST(ERLC) -fi +ERLC=erlc +ERL=erl +ERLANG_ROOT_DIR=$ERL_TOP +AC_SUBST(ERLC) AC_SUBST(WX_BUILDING_INSIDE_ERLSRC) AC_SUBST(ERLANG_ROOT_DIR) @@ -329,7 +317,7 @@ dnl dnl Check for wxwidgets dnl if test "$cross_compiling" = "yes"; then - echo "Cross compilation of the wx driver is not supported yet, wx will NOT be usable" > ./CONF_INFO + echo "Cross compilation of the wx driver is not supported yet, wx will NOT be usable" >> ./CONF_INFO WXERL_CAN_BUILD_DRIVER=false elif test X"$MIXED_CYGWIN_VC" = X"no" -a X"$MIXED_MSYS_VC" = X"no"; then WX_VERSION=`wx-config --version` @@ -398,13 +386,9 @@ define(wx_warn_text,[ wxWidgets version is $reqwx or above.]) if test "$wxWin" != 1; then - if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_ERROR([wx_warn_text]) - else - echo "wxWidgets not found, wx will NOT be usable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([wx_warn_text]) - fi + echo "wxWidgets not found, wx will NOT be usable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([wx_warn_text]) fi else AC_MSG_CHECKING(for wxWidgets in standard locations) @@ -502,13 +486,9 @@ else if test -z "$WX_LIBS_STATIC"; then AC_MSG_RESULT([failed]) - if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_ERROR([Cannot find core lib version for wxWidgets]) - else - echo "No usable wxWidgets not found, wx will not be useable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([Cannot find core lib version for wxWidgets]) - fi + echo "No usable wxWidgets not found, wx will not be useable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Cannot find core lib version for wxWidgets]) fi WX_HAVE_STATIC_LIBS=true AC_SUBST(WX_CFLAGS) @@ -550,8 +530,8 @@ AC_MSG_RESULT($HAVE_GL_SUPPORT) AC_SUBST(HAVE_GL_SUPPORT) if test X"$HAVE_GL_SUPPORT" != X"yes" ; then - echo "wxWidgets don't have gl support, wx will NOT be useable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false + echo "wxWidgets don't have gl support, wx will NOT be useable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false fi dnl Check for GLintptr @@ -610,7 +590,7 @@ dnl AC_CHECK_HEADERS([wx/stc/stc.h], [], [WXERL_CAN_BUILD_DRIVER=false - echo "wxWidgets don't have wxStyledTextControl (stc.h), wx will NOT be useable" > ./CONF_INFO + echo "wxWidgets don't have wxStyledTextControl (stc.h), wx will NOT be useable" >> ./CONF_INFO AC_MSG_WARN([Can not find wx/stc/stc.h $CXXFLAGS]) ], [#ifdef WIN32 @@ -670,9 +650,9 @@ AC_LANG_POP(C++) AC_MSG_RESULT($CAN_LINK_WX) if test X"$CAN_LINK_WX" != X"yes" ; then - echo "Can not link the wx driver, wx will NOT be useable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([Can not link wx program are all developer packages installed?]) + echo "Can not link the wx driver, wx will NOT be useable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Can not link wx program are all developer packages installed?]) fi fi dnl - if test "$WXERL_CAN_BUILD_DRIVER" != "false" @@ -721,6 +701,7 @@ esac AC_SUBST(SO_EXT) AC_SUBST(RUN_ERL) +AC_SUBST(CXXNOOPTFLAGS) if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then @@ -737,12 +718,6 @@ CONFIG_STATUS=$WXERL_SYS_TYPE/config.status dnl -if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - if test X"$WXERL_CAN_BUILD_DRIVER" != X"true" ; then - AC_MSG_ERROR([Cannot build wxErlang driver, see ./CONF_INFO for information]) - fi -fi - AC_CONFIG_FILES([ config.mk c_src/Makefile @@ -750,20 +725,9 @@ AC_CONFIG_FILES([ AC_OUTPUT -if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_NOTICE() - AC_MSG_NOTICE(--------------------------------------------------) - AC_MSG_NOTICE(Using erlang compiler: [$ERLC]) - AC_MSG_NOTICE(wxErlang Install in: [$ERLANG_ROOT_DIR/lib/wx-$ERLWX_VSN]) - AC_MSG_NOTICE(Erlang driver in priv/[$WXERL_SYS_TYPE]/wxe_driver[$SO_EXT]) - AC_MSG_NOTICE(--------------------------------------------------) -fi - -if test X"$WX_BUILDING_INSIDE_ERLSRC" = X"true" ; then - CORES=`ls core* 2>/dev/null` - if test X"$CORES" != X"" ; then - echo "Configure dumped core files" > ignore_core_files - fi +CORES=`ls core* 2>/dev/null` +if test X"$CORES" != X"" ; then + echo "Configure dumped core files" > ignore_core_files fi diff --git a/make/otp_version_tickets_in_merge b/make/otp_version_tickets_in_merge index e69de29bb2..fd9b36720a 100644 --- a/make/otp_version_tickets_in_merge +++ b/make/otp_version_tickets_in_merge @@ -0,0 +1,6 @@ +OTP-15805 +OTP-15819 +OTP-15867 +OTP-15879 +OTP-15887 +OTP-15888 diff --git a/otp_versions.table b/otp_versions.table index d05aff5356..92e04a3035 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-22.0.4 : erts-10.4.3 kernel-6.4.1 ssl-9.3.3 # asn1-5.0.9 common_test-1.17.3 compiler-7.4.2 crypto-4.5.1 debugger-4.2.7 dialyzer-4.0.1 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 stdlib-3.9.2 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0.3 : compiler-7.4.2 dialyzer-4.0.1 erts-10.4.2 ssl-9.3.2 stdlib-3.9.2 # asn1-5.0.9 common_test-1.17.3 crypto-4.5.1 debugger-4.2.7 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0.2 : compiler-7.4.1 crypto-4.5.1 erts-10.4.1 stdlib-3.9.1 # asn1-5.0.9 common_test-1.17.3 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 ssl-9.3.1 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : OTP-22.0.1 : ssl-9.3.1 # asn1-5.0.9 common_test-1.17.3 compiler-7.4 crypto-4.5 debugger-4.2.7 dialyzer-4.0 diameter-2.2.1 edoc-0.11 eldap-1.2.8 erl_docgen-0.9.1 erl_interface-3.12 erts-10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.19 inets-7.0.8 jinterface-1.10 kernel-6.4 megaco-3.18.5 mnesia-4.16 observer-2.9.1 odbc-2.12.4 os_mon-2.5 parsetools-2.1.8 public_key-1.6.7 reltool-0.8 runtime_tools-1.13.3 sasl-3.4 snmp-5.3 ssh-4.7.7 stdlib-3.9 syntax_tools-2.2 tftp-1.0.1 tools-3.2 wx-1.8.8 xmerl-1.3.21 : diff --git a/system/doc/reference_manual/typespec.xml b/system/doc/reference_manual/typespec.xml index 27cd0ba83d..f517259a64 100644 --- a/system/doc/reference_manual/typespec.xml +++ b/system/doc/reference_manual/typespec.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2018</year> + <year>2003</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -140,8 +140,7 @@ | nonempty_improper_list(Type1, Type2) %% Type1 and Type2 as above | nonempty_list(Type) %% Proper non-empty list - Map :: map() %% denotes a map of any size - | #{} %% denotes the empty map + Map :: #{} %% denotes the empty map | #{AssociationList} Tuple :: tuple() %% denotes a tuple of any size @@ -192,17 +191,16 @@ <c>AssociationList</c> are allowed to overlap, and if they do, the leftmost association takes precedence. A map association has a key in <c>AssociationList</c> if it belongs to this type. - <c>AssociationList</c> can contain both mandatory and optional - association types. + <c>AssociationList</c> can contain both mandatory <c>(:=)</c> and + optional <c>(=>)</c> association types. If an association type is mandatory, an association with that type - is to be present. + needs to be present. In the case of an optional association type it is not required for the key type to be present. </p> <p> - Notice that the syntactic representation of <c>map()</c> is - <c>#{any() => any()}</c> (or <c>#{_ => _}</c>), not <c>#{}</c>. The notation <c>#{}</c> specifies the singleton type for the empty map. + Note that this notation is not a shorthand for the <c>map()</c> type. </p> <p> For convenience, the following types are also built-in. @@ -259,6 +257,9 @@ <cell><c>iolist()</c></cell><cell><c>maybe_improper_list(byte() | binary() | iolist(), binary() | [])</c></cell> </row> <row> + <cell><c>map()</c></cell><cell><c>#{any() => any()}</c></cell> + </row> + <row> <cell><c>function()</c></cell><cell><c>fun()</c></cell> </row> <row> |