diff options
24 files changed, 822 insertions, 276 deletions
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index db9c258cb7..602db106b1 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -413,6 +413,7 @@ bif re:compile/1 bif re:compile/2 bif re:run/2 bif re:run/3 +bif re:internal_run/4 # # Bifs in lists module. diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index e0b9202fe7..b3bf1c7ee3 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -46,7 +46,7 @@ static Export *urun_trap_exportp = NULL; static Export *ucompile_trap_exportp = NULL; static BIF_RETTYPE re_exec_trap(BIF_ALIST_3); -static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first); static void *erts_erts_pcre_malloc(size_t size) { return erts_alloc(ERTS_ALC_T_RE_HEAP,size); @@ -1094,7 +1094,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) * The actual re:run/2,3 BIFs */ static BIF_RETTYPE -re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first) { const pcre *code_tmp; RestartContext restart; @@ -1120,6 +1120,14 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) < 0) { BIF_ERROR(p,BADARG); } + if (!first) { + /* + * 'first' is false when re:grun() previously has called re:internal_run() + * with the same subject; i.e., no need to do yet another validation of + * the subject regarding utf8 encoding... + */ + options |= PCRE_NO_UTF8_CHECK; + } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); @@ -1360,15 +1368,28 @@ handle_iolist: } BIF_RETTYPE +re_internal_run_4(BIF_ALIST_4) +{ + int first; + if (BIF_ARG_4 == am_false) + first = 0; + else if (BIF_ARG_4 == am_true) + first = !0; + else + BIF_ERROR(BIF_P,BADARG); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, first); +} + +BIF_RETTYPE re_run_3(BIF_ALIST_3) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, !0); } BIF_RETTYPE re_run_2(BIF_ALIST_2) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL, !0); } /* @@ -1407,6 +1428,7 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3) loop_count = 0xFFFFFFFF; #endif rc = erts_pcre_exec(NULL, &(restartp->extra), NULL, 0, 0, 0, NULL, 0); + ASSERT(loop_count != 0xFFFFFFFF); BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index f58a606d57..55e469b553 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -4051,6 +4051,7 @@ erts_proc_sig_signal_size(ErtsSignal *sig) case ERTS_MON_TYPE_DIST_PROC: case ERTS_MON_TYPE_NODE: size = erts_monitor_size((ErtsMonitor *) sig); + break; default: ERTS_INTERNAL_ERROR("Unexpected sig type"); break; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9d68a4a68b..b9d4f6afcc 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1692,6 +1692,7 @@ i_increment rxy W d # Handle unoptimized code. i_plus S1=c S2=c Fail Dst => move S1 x | i_plus x S2 Fail Dst +i_plus S1=c S2=xy Fail Dst => i_plus S2 S1 Fail Dst i_plus xy xyc j? d diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h index 3563791223..505e2ccce0 100644 --- a/erts/emulator/pcre/pcre.h +++ b/erts/emulator/pcre/pcre.h @@ -240,6 +240,9 @@ with J. */ #define PCRE_UTF8_ERR20 20 #define PCRE_UTF8_ERR21 21 #define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ +#if defined(ERLANG_INTEGRATION) +#define PCRE_UTF8_YIELD 23 +#endif /* Specific error codes for UTF-16 validity checks */ diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 1946e97a72..55a7b377bf 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -6642,10 +6642,16 @@ typedef struct { REAL_PCRE *Xre; heapframe Xframe_zero; /* Always NO_RECURSE */ + /* for yield in valid_utf() */ + + struct PRIV(valid_utf_ystate) valid_utf_ystate; + /* Original function parameters that need be saved */ int Xstart_offset; int Xoffsetcount; int *Xoffsets; + int Xlength; + PCRE_SPTR Xsubject; } PcreExecContext; #endif @@ -6675,6 +6681,7 @@ pcre32_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, #endif { #ifndef ERLANG_INTEGRATION +#define ERTS_UPDATE_CONSUMED(X, MD) int rc, ocount, arg_offset_max; int newline; BOOL using_temporary_offsets = FALSE; @@ -6736,6 +6743,8 @@ heapframe frame_zero; start_offset = exec_context->Xstart_offset; \ offsetcount = exec_context->Xoffsetcount; \ offsets = exec_context->Xoffsets; \ + length = exec_context->Xlength; \ + subject = exec_context->Xsubject; \ } while (0) #define SWAPOUT() do { \ @@ -6750,8 +6759,30 @@ heapframe frame_zero; exec_context->Xstart_offset = start_offset; \ exec_context->Xoffsetcount = offsetcount; \ exec_context->Xoffsets = offsets; \ + exec_context->Xlength = length; \ + exec_context->Xsubject = subject; \ } while (0) +#define ERTS_UPDATE_CONSUMED(X, MD) \ +do { \ + if (((X)->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { \ + unsigned long consumed__; \ + if (!(X)->restart_data) { \ + consumed__ = 0; \ + } \ + else { \ + PcreExecContext *ctx__ = (PcreExecContext *) \ + (*(X)->restart_data); \ + consumed__ = ctx__->valid_utf_ystate.cnt; \ + ctx__->valid_utf_ystate.cnt = 0; \ + } \ + if ((MD)) { \ + match_data *md__ = (MD); \ + consumed__ += (X)->loop_limit - md__->loop_limit; \ + } \ + *((X)->loop_counter_return) = consumed__; \ + } \ +} while (0) PcreExecContext *exec_context; PcreExecContext internal_context; @@ -6776,15 +6807,21 @@ pcre_uchar req_char; /* we are restarting, every initialization is skipped and we jump directly into the loop */ exec_context = (PcreExecContext *) *(extra_data->restart_data); SWAPIN(); - + if (exec_context->valid_utf_ystate.yielded) + goto restart_valid_utf; goto RESTART_INTERRUPTED; } else { if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { exec_context = (PcreExecContext *) (erts_pcre_malloc)(sizeof(PcreExecContext)); - *(extra_data->restart_data) = (void *) exec_context; + *(extra_data->restart_data) = (void *) exec_context; + exec_context->valid_utf_ystate.yielded = 0; /* need freeing by special routine from client */ } else { +#if defined(ERLANG_INTEGRATION) + fprintf(stderr, "Unexpected execution path\n"); + abort(); +#endif exec_context = &internal_context; } @@ -6865,9 +6902,38 @@ code for an invalid string if a results vector is available. */ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) { int erroroffset; - int errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length, &erroroffset); + int errorcode; + +#if !defined(ERLANG_INTEGRATION) + errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length); +#else + struct PRIV(valid_utf_ystate) *ystate; + + if (!extra_data || !extra_data->restart_data) { + ystate = NULL; + } + else if (!(extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { + exec_context->valid_utf_ystate.cnt = 10; + ystate = NULL; + } + else { + exec_context->valid_utf_ystate.yielded = 0; + restart_valid_utf: + ystate = &exec_context->valid_utf_ystate; + ystate->cnt = (int) extra_data->loop_limit; + } + errorcode = PRIV(yielding_valid_utf)((PCRE_PUCHAR)subject, length, + &erroroffset, ystate); +#endif if (errorcode != 0) { +#if defined(ERLANG_INTEGRATION) + if (ystate && ystate->yielded) { + ERTS_UPDATE_CONSUMED(extra_data, NULL); + SWAPOUT(); + return PCRE_ERROR_LOOP_LIMIT; + } +#endif if (offsetcount >= 2) { offsets[0] = erroroffset; @@ -6890,6 +6956,11 @@ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) return PCRE_ERROR_BADUTF8_OFFSET; #endif } +#if defined(ERLANG_INTEGRATION) +else { + exec_context->valid_utf_ystate.cnt = 0; +} +#endif #endif /* If the pattern was successfully studied with JIT support, run the JIT @@ -6950,7 +7021,11 @@ if (extra_data != NULL) #ifdef ERLANG_INTEGRATION if ((flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { - md->loop_limit = extra_data->loop_limit; + md->loop_limit = extra_data->loop_limit; + if (extra_data->restart_data) + md->loop_limit -= extra_data->loop_limit - exec_context->valid_utf_ystate.cnt; + if (md->loop_limit < 10) + md->loop_limit = 10; /* At least do something if we've come this far... */ } #endif } @@ -7266,14 +7341,8 @@ for(;;) #endif if ((start_bits[c/8] & (1 << (c&7))) != 0) { -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif - break; + ERTS_UPDATE_CONSUMED(extra_data, md); + break; } start_match++; } @@ -7298,13 +7367,7 @@ for(;;) (pcre_uint32)(end_subject - start_match) < study->minlength) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7353,13 +7416,7 @@ for(;;) if (p >= end_subject) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7390,11 +7447,7 @@ for(;;) EDEBUGF(("Calling match...")); rc = match(start_match, md->start_code, start_match, 2, md, NULL, 0); #ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } + ERTS_UPDATE_CONSUMED(extra_data, md); SWAPOUT(); while(rc == PCRE_ERROR_LOOP_LIMIT) { EDEBUGF(("Loop limit break detected")); diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h index c84dcb5a38..71f473e86f 100644 --- a/erts/emulator/pcre/pcre_internal.h +++ b/erts/emulator/pcre/pcre_internal.h @@ -2756,6 +2756,17 @@ extern int PRIV(strcmp_uc_c8_utf)(const pcre_uchar *, #endif /* COMPILE_PCRE[8|16|32] */ +#if defined(ERLANG_INTEGRATION) +struct PRIV(valid_utf_ystate) { + unsigned int cnt; + int length; + int yielded; + PCRE_PUCHAR p; +}; +extern int PRIV(yielding_valid_utf)(PCRE_PUCHAR, int, int *, + struct PRIV(valid_utf_ystate) *); +#endif + extern const pcre_uchar *PRIV(find_bracket)(const pcre_uchar *, BOOL, int); extern BOOL PRIV(is_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, int *, BOOL); diff --git a/erts/emulator/pcre/pcre_valid_utf8.c b/erts/emulator/pcre/pcre_valid_utf8.c index 516d8f4725..1dc1f9ba0c 100644 --- a/erts/emulator/pcre/pcre_valid_utf8.c +++ b/erts/emulator/pcre/pcre_valid_utf8.c @@ -107,19 +107,80 @@ Returns: = 0 if the string is a valid UTF-8 string int PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) { + +#if defined(ERLANG_INTEGRATION) + return PRIV(yielding_valid_utf)(string, length, erroroffset, NULL); +} + +int +PRIV(yielding_valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset, struct PRIV(valid_utf_ystate) *ystate) +{ +#endif + #ifdef SUPPORT_UTF register PCRE_PUCHAR p; +#if defined(ERLANG_INTEGRATION) +register long cnt; + +if (!ystate) { + cnt = -1; +} +else { + cnt = ystate->cnt; + if (ystate->yielded) { + p = ystate->p; + length = ystate->length; + if (length < 0) + goto restart_length; + else + goto restart_validate; + } +} +#endif + if (length < 0) { - for (p = string; *p != 0; p++); - length = (int)(p - string); + for (p = string; *p != 0; p++) { +#if defined(ERLANG_INTEGRATION) + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + restart_length: + (void) !0; +#endif + } + length = (int)(p - string); } for (p = string; length-- > 0; p++) { register pcre_uchar ab, c, d; +#if defined(ERLANG_INTEGRATION) + + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + + restart_validate: + +#endif + c = *p; if (c < 128) continue; /* ASCII character */ @@ -290,6 +351,14 @@ for (p = string; length-- > 0; p++) } } +#if defined(ERLANG_INTEGRATION) +if (ystate) { + /* Return with cnt set to amount consumed... */ + ystate->cnt -= cnt; + ystate->yielded = 0; +} +#endif + #else /* Not SUPPORT_UTF */ (void)(string); /* Keep picky compilers happy */ (void)(length); diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 55b1162cfb..b48be3dd04 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -37,10 +37,13 @@ -export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1, + procs_bug/1, ets_count/1, atom_count/1, system_logger/1]). -export([init/1, handle_event/2, handle_call/2]). +-export([init_per_testcase/2, end_per_testcase/2]). + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 2}}]. @@ -48,8 +51,20 @@ suite() -> all() -> [process_count, system_version, misc_smoke_tests, ets_count, heap_size, wordsize, memory, ets_limit, atom_limit, atom_count, + procs_bug, system_logger]. + +init_per_testcase(procs_bug, Config) -> + procs_bug(init_per_testcase, Config); +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(procs_bug, Config) -> + procs_bug(end_per_testcase, Config); +end_per_testcase(_, _) -> + ok. + %%% %%% The test cases ------------------------------------------------------------- %%% @@ -654,3 +669,41 @@ handle_call(Msg, State) -> handle_event(Event, State) -> State ! {report_handler, Event}, {ok, State}. + + +%% OTP-15909: Provoke bug that would cause VM crash +%% if doing system_info(procs) when process have queued exit/down signals. +procs_bug(init_per_testcase, Config) -> + %% Use single scheduler and process prio to starve monitoring processes + %% from handling their received DOWN signals. + OldSchedOnline = erlang:system_flag(schedulers_online,1), + [{schedulers_online, OldSchedOnline} | Config]; +procs_bug(end_per_testcase, Config) -> + erlang:system_flag(schedulers_online, + proplists:get_value(schedulers_online, Config)), + ok. + +procs_bug(Config) when is_list(Config) -> + {Monee,_} = spawn_opt(fun () -> receive die -> ok end end, + [monitor,{priority,max}]), + Papa = self(), + Pids = [begin + P = spawn_opt(fun () -> + erlang:monitor(process, Monee), + Papa ! {self(),ready}, + receive "nada" -> no end + end, + [link, {priority,normal}]), + {P, ready} = receive M -> M end, + P + end + || _ <- lists:seq(1,10)], + process_flag(priority,high), + Monee ! die, + {'DOWN',_,process,Monee,normal} = receive M -> M end, + + %% This call did crash VM as Pids have pending DOWN signals. + erlang:system_info(procs), + process_flag(priority,normal), + [begin unlink(P), exit(P, kill) end || P <- Pids], + ok. diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 7a5d31b9eb..710a7a9ef6 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -292,11 +292,20 @@ if [ "x$GDB" = "x" ]; then valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log" fi fi - if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then - valgrind_misc_flags="--show-possibly-lost=no --child-silent-after-fork=yes\ - --suppressions=$ROOTDIR/erts/emulator/valgrind/suppress.standard" - else - valgrind_misc_flags="$VALGRIND_MISC_FLAGS" + # Add default flags + vgflags=$VALGRIND_MISC_FLAGS + if [ "x${vgflags#*--show-possibly-lost}" = "x$vgflags" ]; then + vgflags="$vgflags --show-possibly-lost=no" + fi + if [ "x${vgflags#*--child-silent-after-fork}" = "x$vgflags" ]; then + vgflags="$vgflags --child-silent-after-fork=yes" + fi + if [ "x${vgflags#*--suppressions}" = "x$vgflags" ]; then + if [ "x$ERL_TOP" != "x" ]; then + vgflags="$vgflags --suppressions=$ERL_TOP/erts/emulator/valgrind/suppress.standard" + else + echo "No valgrind suppression file found in \$VALGRIND_MISC_FLAGS and \$ERL_TOP not set." + fi fi if which taskset > /dev/null && test -e /proc/cpuinfo; then # We only let valgrind utilize one core with "taskset 1" as it can be very slow @@ -311,7 +320,7 @@ if [ "x$GDB" = "x" ]; then sched_arg= fi - exec $taskset1 valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" + exec $taskset1 valgrind $valgrind_xml $valgrind_log $vgflags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" elif [ $run_rr = yes ]; then if [ $1 = replay ]; then diff --git a/lib/snmp/src/agent/snmpa_set.erl b/lib/snmp/src/agent/snmpa_set.erl index 9833d6fdcc..b3a3bf0ab0 100644 --- a/lib/snmp/src/agent/snmpa_set.erl +++ b/lib/snmp/src/agent/snmpa_set.erl @@ -163,10 +163,14 @@ set_phase_two(MyVarbinds, SubagentVarbinds) -> [MyVarbinds, SubagentVarbinds]), case snmpa_set_lib:try_set(MyVarbinds) of {noError, 0} -> + ?vtrace("set phase two: (local) varbinds set ok", []), set_phase_two_subagents(SubagentVarbinds); - {ErrorStatus, Index} -> + {ErrorStatus, ErrorIndex} -> + ?vlog("set phase two: (local) varbinds set failed" + "~n ErrorStatus: ~p" + "~n ErrorIndex: ~p", [ErrorStatus, ErrorIndex]), set_phase_two_undo_subagents(SubagentVarbinds), - {ErrorStatus, Index} + {ErrorStatus, ErrorIndex} end. %%----------------------------------------------------------------- @@ -188,6 +192,7 @@ set_phase_two_subagents([{SubAgentPid, SAVbs} | SubagentVarbinds]) -> {_SAOids, Vbs} = sa_split(SAVbs), case catch snmpa_agent:subagent_set(SubAgentPid, [phase_two, set, Vbs]) of {noError, 0} -> + ?vtrace("set phase two: subagent ~p varbinds set ok", [SubAgentPid]), set_phase_two_subagents(SubagentVarbinds); {'EXIT', Reason} -> user_err("Lost contact with subagent (set)~n~w. Using genErr", @@ -195,10 +200,14 @@ set_phase_two_subagents([{SubAgentPid, SAVbs} | SubagentVarbinds]) -> set_phase_two_undo_subagents(SubagentVarbinds), {genErr, 0}; {ErrorStatus, ErrorIndex} -> + ?vlog("set phase two: subagent ~p varbinds set failed" + "~n ErrorStatus: ~p" + "~n ErrorIndex: ~p", [SubAgentPid, ErrorStatus, ErrorIndex]), set_phase_two_undo_subagents(SubagentVarbinds), {ErrorStatus, ErrorIndex} end; set_phase_two_subagents([]) -> + ?vtrace("set phase two: subagent(s) set ok", []), {noError, 0}. %%----------------------------------------------------------------- diff --git a/lib/snmp/test/snmp_agent_test.erl b/lib/snmp/test/snmp_agent_test.erl index 71e3fa3b9a..860ca17cdb 100644 --- a/lib/snmp/test/snmp_agent_test.erl +++ b/lib/snmp/test/snmp_agent_test.erl @@ -667,22 +667,39 @@ init_per_group(GroupName, Config) -> snmp_test_lib:init_group_top_dir(GroupName, Config). init_per_group_ipv6(GroupName, Config, Init) -> - {ok, Hostname0} = inet:gethostname(), - case ct:require(ipv6_hosts) of - ok -> - case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of - true -> - Init( - snmp_test_lib:init_group_top_dir( - GroupName, - [{ipfamily, inet6}, - {ip, ?LOCALHOST(inet6)} - | lists:keydelete(ip, 1, Config)])); - false -> - {skip, "Host does not support IPV6"} - end; - _ -> - {skip, "Test config ipv6_hosts is missing"} + %% <OS-CONDITIONAL-SKIP> + %% This is a higly questionable test. + %% But until we have time to figure out what IPv6 issues + %% are actually causing the failures... + OSSkipable = [{unix, + [ + {darwin, fun(V) when (V > {9, 8, 0}) -> + %% This version is OK: No Skip + false; + (_) -> + %% This version is *not* ok: Skip + true + end} + ] + }], + %% </OS-CONDITIONAL-SKIP> + case ?OS_BASED_SKIP(OSSkipable) of + true -> + {skip, "Host *may* not *properly* support IPV6"}; + false -> + %% Even if this host supports IPv6 we don't use it unless its + %% one of the configured/supported IPv6 hosts... + case (?HAS_SUPPORT_IPV6() andalso ?IS_IPV6_HOST()) of + true -> + Init( + snmp_test_lib:init_group_top_dir( + GroupName, + [{ipfamily, inet6}, + {ip, ?LOCALHOST(inet6)} + | lists:keydelete(ip, 1, Config)])); + false -> + {skip, "Host does not support IPv6"} + end end. end_per_group(all_tcs, Config) -> diff --git a/lib/snmp/test/snmp_agent_test_lib.erl b/lib/snmp/test/snmp_agent_test_lib.erl index 6defdadb5a..1128fc8a8c 100644 --- a/lib/snmp/test/snmp_agent_test_lib.erl +++ b/lib/snmp/test/snmp_agent_test_lib.erl @@ -66,7 +66,7 @@ ]). %% Internal exports --export([wait/5, run/4]). +-export([tc_wait/5, tc_run/4]). -include_lib("kernel/include/file.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -276,36 +276,94 @@ init_case(Config) when is_list(Config) -> %%% configuration. %%%-------------------------------------------------- -try_test(Mod, Func) -> - call(get(mgr_node), ?MODULE, run, [Mod, Func, [], []]). - -try_test(Mod, Func, A) -> - call(get(mgr_node), ?MODULE, run, [Mod, Func, A, []]). - -try_test(Mod, Func, A, Opts) -> - call(get(mgr_node), ?MODULE, run, [Mod, Func, A, Opts]). - -call(N,M,F,A) -> - ?DBG("call -> entry with~n" - " N: ~p~n" - " M: ~p~n" - " F: ~p~n" - " A: ~p~n" - " when~n" - " get(): ~p", - [N,M,F,A,get()]), - spawn(N, ?MODULE, wait, [self(),get(),M,F,A]), +try_test(TcRunMod, TcRunFunc) -> + try_test(TcRunMod, TcRunFunc, []). + +try_test(TcRunMod, TcRunFunc, TcRunArgs) -> + try_test(TcRunMod, TcRunFunc, TcRunArgs, []). + +try_test(TcRunMod, TcRunFunc, TcRunArgs, TcRunOpts) -> + Node = get(mgr_node), + Mod = ?MODULE, + Func = tc_run, + Args = [TcRunMod, TcRunFunc, TcRunArgs, TcRunOpts], + tc_try(Node, Mod, Func, Args). + +%% We spawn a test case runner process on the manager node. +%% The assumption is that the manager shall do something, but +%% not all test cases have the manager perform actions. +%% In some cases we make a rpc call back to the agent node directly +%% and call something in the agent... (for example the info_test +%% test case). +%% We should use link (instead of monitor) in order for the test case +%% timeout cleanup (kills) should have effect on the test case runner +%% process as well. + +tc_try(N, M, F, A) -> + ?PRINT2("tc_try -> entry with" + "~n N: ~p" + "~n M: ~p" + "~n F: ~p" + "~n A: ~p" + "~n when" + "~n get(): ~p" + "~n", [N, + M, F, A, + get()]), + case net_adm:ping(N) of + pong -> + ?PRINT2("tc_try -> ~p still running - start runner~n", [N]), + OldFlag = trap_exit(true), % Make sure we catch it + Runner = spawn_link(N, ?MODULE, tc_wait, [self(), get(), M, F, A]), + await_tc_runner_started(Runner, OldFlag), + await_tc_runner_done(Runner, OldFlag); + pang -> + ?EPRINT2("tc_try -> ~p *not* running~n", [N]), + exit({node_not_running, N}) + end. + +await_tc_runner_started(Runner, OldFlag) -> + ?PRINT2("await tc-runner (~p) start ack~n", [Runner]), + receive + {'EXIT', Runner, Reason} -> + ?EPRINT2("TC runner start failed: " + "~n ~p~n", [Reason]), + exit({tx_runner_start_failed, Reason}); + {tc_runner_started, Runner} -> + ?PRINT2("TC runner start acknowledged~n"), + ok + after 10000 -> + trap_exit(OldFlag), + unlink_and_flush_exit(Runner), + RunnerInfo = process_info(Runner), + ?EPRINT2("TC runner start timeout: " + "~n ~p", [RunnerInfo]), + %% If we don't get a start ack within 10 seconds, we are f*ed + exit(Runner, kill), + exit({tc_runner_start, timeout, RunnerInfo}) + end. + +await_tc_runner_done(Runner, OldFlag) -> receive - {done, {'EXIT', Rn}, Loc} -> - ?DBG("call -> done with exit: " - "~n Rn: ~p" - "~n Loc: ~p", [Rn, Loc]), + {'EXIT', Runner, Reason} -> + ?EPRINT2("TC runner failed: " + "~n ~p~n", [Reason]), + exit({tx_runner_failed, Reason}); + {tc_runner_done, Runner, {'EXIT', Rn}, Loc} -> + ?PRINT2("call -> done with exit: " + "~n Rn: ~p" + "~n Loc: ~p" + "~n", [Rn, Loc]), + trap_exit(OldFlag), + unlink_and_flush_exit(Runner), put(test_server_loc, Loc), exit(Rn); - {done, Ret, _Zed} -> + {tc_runner_done, Runner, Ret, _Zed} -> ?DBG("call -> done:" "~n Ret: ~p" "~n Zed: ~p", [Ret, _Zed]), + trap_exit(OldFlag), + unlink_and_flush_exit(Runner), case Ret of {error, Reason} -> exit(Reason); @@ -314,49 +372,66 @@ call(N,M,F,A) -> end end. -wait(From, Env, M, F, A) -> - ?DBG("wait -> entry with" - "~n From: ~p" - "~n Env: ~p" - "~n M: ~p" - "~n F: ~p" - "~n A: ~p", [From, Env, M, F, A]), +trap_exit(Flag) when is_boolean(Flag) -> + erlang:process_flag(trap_exit, Flag). + +unlink_and_flush_exit(Pid) -> + unlink(Pid), + receive + {'EXIT', Pid, _} -> + ok + after 0 -> + ok + end. + +tc_wait(From, Env, M, F, A) -> + ?PRINT2("tc_wait -> entry with" + "~n From: ~p" + "~n Env: ~p" + "~n M: ~p" + "~n F: ~p" + "~n A: ~p", [From, Env, M, F, A]), + From ! {tc_runner_started, self()}, lists:foreach(fun({K,V}) -> put(K,V) end, Env), - Rn = (catch apply(M, F, A)), - ?DBG("wait -> Rn: ~n~p", [Rn]), - From ! {done, Rn, get(test_server_loc)}, - exit(Rn). - -run(Mod, Func, Args, Opts) -> - ?DBG("run -> entry with" - "~n Mod: ~p" - "~n Func: ~p" - "~n Args: ~p" - "~n Opts: ~p", [Mod, Func, Args, Opts]), - M = get(mib_dir), - Dir = get(mgr_dir), - User = snmp_misc:get_option(user, Opts, "all-rights"), - SecLevel = snmp_misc:get_option(sec_level, Opts, noAuthNoPriv), - EngineID = snmp_misc:get_option(engine_id, Opts, "agentEngine"), + ?PRINT2("tc_wait -> env set - now run tc~n"), + Res = (catch apply(M, F, A)), + ?PRINT2("tc_wait -> tc run done: " + "~n ~p" + "~n", [Res]), + From ! {tc_runner_done, self(), Res, get(test_server_loc)}, + exit(Res). + +tc_run(Mod, Func, Args, Opts) -> + ?PRINT2("tc_run -> entry with" + "~n Mod: ~p" + "~n Func: ~p" + "~n Args: ~p" + "~n Opts: ~p" + "~n", [Mod, Func, Args, Opts]), + (catch snmp_test_mgr:stop()), % If we had a running mgr from a failed case + M = get(mib_dir), + Dir = get(mgr_dir), + User = snmp_misc:get_option(user, Opts, "all-rights"), + SecLevel = snmp_misc:get_option(sec_level, Opts, noAuthNoPriv), + EngineID = snmp_misc:get_option(engine_id, Opts, "agentEngine"), CtxEngineID = snmp_misc:get_option(context_engine_id, Opts, EngineID), - Community = snmp_misc:get_option(community, Opts, "all-rights"), - ?DBG("run -> start crypto app",[]), - _CryptoRes = ?CRYPTO_START(), - ?DBG("run -> Crypto: ~p", [_CryptoRes]), - catch snmp_test_mgr:stop(), % If we had a running mgr from a failed case - StdM = join(code:priv_dir(snmp), "mibs") ++ "/", - Vsn = get(vsn), - ?DBG("run -> config:" - "~n M: ~p" - "~n Vsn: ~p" - "~n Dir: ~p" - "~n User: ~p" - "~n SecLevel: ~p" - "~n EngineID: ~p" - "~n CtxEngineID: ~p" - "~n Community: ~p" - "~n StdM: ~p", - [M,Vsn,Dir,User,SecLevel,EngineID,CtxEngineID,Community,StdM]), + Community = snmp_misc:get_option(community, Opts, "all-rights"), + ?DBG("tc_run -> start crypto app",[]), + _CryptoRes = ?CRYPTO_START(), + ?DBG("tc_run -> Crypto: ~p", [_CryptoRes]), + StdM = join(code:priv_dir(snmp), "mibs") ++ "/", + Vsn = get(vsn), + ?PRINT2("tc_run -> config:" + "~n M: ~p" + "~n Vsn: ~p" + "~n Dir: ~p" + "~n User: ~p" + "~n SecLevel: ~p" + "~n EngineID: ~p" + "~n CtxEngineID: ~p" + "~n Community: ~p" + "~n StdM: ~p" + "~n", [M,Vsn,Dir,User,SecLevel,EngineID,CtxEngineID,Community,StdM]), case snmp_test_mgr:start([%% {agent, snmp_test_lib:hostname()}, {packet_server_debug, true}, {debug, true}, @@ -377,23 +452,23 @@ run(Mod, Func, Args, Opts) -> {ok, _Pid} -> case (catch apply(Mod, Func, Args)) of {'EXIT', Reason} -> - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), ?FAIL({apply_failed, {Mod, Func, Args}, Reason}); Res -> - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), Res end; {error, Reason} -> ?EPRINT2("Failed starting (test) manager: " "~n ~p", [Reason]), - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), ?line ?FAIL({mgr_start_error, Reason}); Err -> ?EPRINT2("Failed starting (test) manager: " "~n ~p", [Err]), - catch snmp_test_mgr:stop(), + (catch snmp_test_mgr:stop()), ?line ?FAIL({mgr_start_failure, Err}) end. diff --git a/lib/snmp/test/snmp_manager_test.erl b/lib/snmp/test/snmp_manager_test.erl index 5b0ebf8647..d959d9e09b 100644 --- a/lib/snmp/test/snmp_manager_test.erl +++ b/lib/snmp/test/snmp_manager_test.erl @@ -619,38 +619,47 @@ init_per_group(event_tests_mt = GroupName, Config) -> GroupName, [{manager_net_if_module, snmpm_net_if_mt} | Config]); init_per_group(ipv6_mt = GroupName, Config) -> - {ok, Hostname0} = inet:gethostname(), - case ct:require(ipv6_hosts) of - ok -> - case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of - true -> - ipv6_init( - snmp_test_lib:init_group_top_dir( - GroupName, - [{manager_net_if_module, snmpm_net_if_mt} - | Config])); - false -> - {skip, "Host does not support IPv6"} - end; - _ -> - {skip, "Test config ipv6_hosts is missing"} - end; + init_per_group_ipv6(GroupName, + [{manager_net_if_module, snmpm_net_if_mt} | Config]); init_per_group(ipv6 = GroupName, Config) -> - {ok, Hostname0} = inet:gethostname(), - case ct:require(ipv6_hosts) of - ok -> - case lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) of - true -> - ipv6_init(snmp_test_lib:init_group_top_dir(GroupName, Config)); - false -> - {skip, "Host does not support IPv6"} - end; - _ -> - {skip, "Test config ipv6_hosts is missing"} - end; + init_per_group_ipv6(GroupName, Config); init_per_group(GroupName, Config) -> snmp_test_lib:init_group_top_dir(GroupName, Config). + +init_per_group_ipv6(GroupName, Config) -> + %% <OS-CONDITIONAL-SKIP> + OSSkipable = [{unix, + [ + {darwin, fun(V) when (V > {9, 8, 0}) -> + %% This version is OK: No Skip + false; + (_) -> + %% This version is *not* ok: Skip + %% We need a fully qualified hostname + %% to get a proper IPv6 address (in this + %% version), but its just to messy, so + %% instead we skip this **OLD** darwin... + true + end} + ] + }], + %% </OS-CONDITIONAL-SKIP> + case ?OS_BASED_SKIP(OSSkipable) of + true -> + {skip, "Host *may* not *properly* support IPV6"}; + false -> + %% Even if this host supports IPv6 we don't use it unless its + %% one of the configures/supported IPv6 hosts... + case (?HAS_SUPPORT_IPV6() andalso ?IS_IPV6_HOST()) of + true -> + ipv6_init(snmp_test_lib:init_group_top_dir(GroupName, Config)); + false -> + {skip, "Host does not support IPv6"} + end + end. + + end_per_group(_GroupName, Config) -> %% Do we really need to do this? lists:keydelete(snmp_group_top_dir, 1, Config). diff --git a/lib/snmp/test/snmp_test_lib.erl b/lib/snmp/test/snmp_test_lib.erl index a483690653..42f710e4cd 100644 --- a/lib/snmp/test/snmp_test_lib.erl +++ b/lib/snmp/test/snmp_test_lib.erl @@ -25,7 +25,9 @@ -export([hostname/0, hostname/1, localhost/0, localhost/1, os_type/0, sz/1, display_suite_info/1]). --export([non_pc_tc_maybe_skip/4, os_based_skip/1]). +-export([non_pc_tc_maybe_skip/4, os_based_skip/1, + has_support_ipv6/0, has_support_ipv6/1, + is_ipv6_host/0, is_ipv6_host/1]). -export([fix_data_dir/1, init_suite_top_dir/2, init_group_top_dir/2, init_testcase_top_dir/2, lookup/2, @@ -52,11 +54,12 @@ hostname() -> hostname(node()). hostname(Node) -> - from($@, atom_to_list(Node)). - -from(H, [H | T]) -> T; -from(H, [_ | T]) -> from(H, T); -from(_H, []) -> []. + case string:tokens(atom_to_list(Node), [$@]) of + [_, Host] -> + Host; + _ -> + [] + end. %% localhost() -> %% {ok, Ip} = snmp_misc:ip(net_adm:localhost()), @@ -78,7 +81,9 @@ localhost(Family) -> {error, Reason1} -> fail({getifaddrs, Reason1}, ?MODULE, ?LINE) end; - {ok, {0, _, _, _, _, _, _, _}} when (Family =:= inet6) -> + {ok, {A1, _, _, _, _, _, _, _}} when (Family =:= inet6) andalso + ((A1 =:= 0) orelse + (A1 =:= 16#fe80)) -> %% Ouch, we need to use something else case inet:getifaddrs() of {ok, IfList} -> @@ -207,53 +212,108 @@ non_pc_tc_maybe_skip(Config, Condition, File, Line) end. +%% The type and spec'ing is just to increase readability +-type os_family() :: win32 | unix. +-type os_name() :: atom(). +-type os_version() :: string() | {non_neg_integer(), + non_neg_integer(), + non_neg_integer()}. +-type os_skip_check() :: fun(() -> boolean()) | + fun((os_version()) -> boolean()). +-type skippable() :: any | [os_family() | + {os_family(), os_name() | + [os_name() | {os_name(), + os_skip_check()}]}]. + +-spec os_based_skip(skippable()) -> boolean(). + os_based_skip(any) -> - io:format("os_based_skip(any) -> entry" - "~n", []), true; os_based_skip(Skippable) when is_list(Skippable) -> - io:format("os_based_skip -> entry with" - "~n Skippable: ~p" - "~n", [Skippable]), - {OsFam, OsName} = - case os:type() of - {_Fam, _Name} = FamAndName -> - FamAndName; - Fam -> - {Fam, undefined} - end, - io:format("os_based_skip -> os-type: " - "~n OsFam: ~p" - "~n OsName: ~p" - "~n", [OsFam, OsName]), + os_base_skip(Skippable, os:type()); +os_based_skip(_Crap) -> + false. + +os_base_skip(Skippable, {OsFam, OsName}) -> + os_base_skip(Skippable, OsFam, OsName); +os_base_skip(Skippable, OsFam) -> + os_base_skip(Skippable, OsFam, undefined). + +os_base_skip(Skippable, OsFam, OsName) -> + %% Check if the entire family is to be skipped + %% Example: [win32, unix] case lists:member(OsFam, Skippable) of true -> true; false -> - case lists:keysearch(OsFam, 1, Skippable) of - {value, {OsFam, OsName}} -> - true; - {value, {OsFam, OsNames}} when is_list(OsNames) -> + %% Example: [{unix, freebsd}] | [{unix, [freebsd, darwin]}] + case lists:keysearch(OsFam, 1, Skippable) of + {value, {OsFam, OsName}} -> + true; + {value, {OsFam, OsNames}} when is_list(OsNames) -> + %% OsNames is a list of: + %% [atom()|{atom(), function/0 | function/1}] case lists:member(OsName, OsNames) of true -> true; false -> - case lists:keymember(OsName, 1, OsNames) of - {value, {OsName, Check}} when is_function(Check) -> - Check(); - _ -> - false - end + os_based_skip_check(OsName, OsNames) end; - _ -> - false - end - end; -os_based_skip(_Crap) -> - io:format("os_based_skip -> entry with" - "~n _Crap: ~p" - "~n", [_Crap]), - false. + _ -> + false + end + end. + +%% Performs a check via a provided fun with arity 0 or 1. +%% The argument is the result of os:version(). +os_based_skip_check(OsName, OsNames) -> + case lists:keysearch(OsName, 1, OsNames) of + {value, {OsName, Check}} when is_function(Check, 0) -> + Check(); + {value, {OsName, Check}} when is_function(Check, 1) -> + Check(os:version()); + _ -> + false + end. + + +%% A basic test to check if current host supports IPv6 +has_support_ipv6() -> + case inet:gethostname() of + {ok, Hostname} -> + has_support_ipv6(Hostname); + _ -> + false + end. + +has_support_ipv6(Hostname) -> + case inet:getaddr(Hostname, inet6) of + {ok, Addr} when (size(Addr) =:= 8) andalso + (element(1, Addr) =/= 0) andalso + (element(1, Addr) =/= 16#fe80) -> + true; + {ok, _} -> + false; + {error, _} -> + false + end. + + +is_ipv6_host() -> + case inet:gethostname() of + {ok, Hostname} -> + is_ipv6_host(Hostname); + {error, _} -> + false + end. + +is_ipv6_host(Hostname) -> + case ct:require(ipv6_hosts) of + ok -> + lists:member(list_to_atom(Hostname), ct:get_config(ipv6_hosts)); + _ -> + false + end. %% ---------------------------------------------------------------- diff --git a/lib/snmp/test/snmp_test_lib.hrl b/lib/snmp/test/snmp_test_lib.hrl index 335f3fff3c..c66602b779 100644 --- a/lib/snmp/test/snmp_test_lib.hrl +++ b/lib/snmp/test/snmp_test_lib.hrl @@ -49,8 +49,12 @@ snmp_test_lib:os_based_skip(Skippable)). -define(NON_PC_TC_MAYBE_SKIP(Config, Condition), snmp_test_lib:non_pc_tc_maybe_skip(Config, Condition, ?MODULE, ?LINE)). --define(SKIP(Reason), snmp_test_lib:skip(Reason, ?MODULE, ?LINE)). --define(FAIL(Reason), snmp_test_lib:fail(Reason, ?MODULE, ?LINE)). +-define(SKIP(Reason), snmp_test_lib:skip(Reason, ?MODULE, ?LINE)). +-define(FAIL(Reason), snmp_test_lib:fail(Reason, ?MODULE, ?LINE)). +-define(IS_IPV6_HOST(), snmp_test_lib:is_ipv6_host()). +-define(IS_IPV6_HOST(H), snmp_test_lib:is_ipv6_host(H)). +-define(HAS_SUPPORT_IPV6(), snmp_test_lib:has_support_ipv6()). +-define(HAS_SUPPORT_IPV6(H), snmp_test_lib:has_support_ipv6(H)). %% - Time macros - @@ -150,8 +154,10 @@ snmp_test_lib:print(P, ?MODULE, ?LINE, F, A)). -define(PRINT1(F, A), snmp_test_lib:print1(F, A)). +-define(PRINT1(F), ?PRINT1(F, [])). -define(EPRINT1(F, A), ?PRINT1("<ERROR> " ++ F, A)). -define(PRINT2(F, A), snmp_test_lib:print2(F, A)). +-define(PRINT2(F), ?PRINT2(F, [])). -define(EPRINT2(F, A), ?PRINT2("<ERROR> " ++ F, A)). diff --git a/lib/snmp/test/snmp_test_mgr.erl b/lib/snmp/test/snmp_test_mgr.erl index 9190c07e6d..9d6be65088 100644 --- a/lib/snmp/test/snmp_test_mgr.erl +++ b/lib/snmp/test/snmp_test_mgr.erl @@ -247,10 +247,21 @@ init({Options, CallerPid}) -> IpFamily = get_value(ipfamily, Options, inet), print("[~w] ~p -> IpFamily: ~p~n", [?MODULE, self(), IpFamily]), AgIp = case snmp_misc:assq(agent, Options) of - {value, Tuple4} when is_tuple(Tuple4) andalso - (size(Tuple4) =:= 4) -> - Tuple4; + {value, Addr} when is_tuple(Addr) andalso + (size(Addr) =:= 4) andalso + (IpFamily =:= inet) -> + print("[~w] ~p -> Addr: ~p~n", + [?MODULE, self(), Addr]), + Addr; + {value, Addr} when is_tuple(Addr) andalso + (size(Addr) =:= 8) andalso + (IpFamily =:= inet6) -> + print("[~w] ~p -> Addr: ~p~n", + [?MODULE, self(), Addr]), + Addr; {value, Host} when is_list(Host) -> + print("[~w] ~p -> Host: ~p~n", + [?MODULE, self(), Host]), {ok, Ip} = snmp_misc:ip(Host, IpFamily), Ip end, diff --git a/lib/snmp/test/snmp_test_mgr_misc.erl b/lib/snmp/test/snmp_test_mgr_misc.erl index 315e3ebd9e..6608a88c00 100644 --- a/lib/snmp/test/snmp_test_mgr_misc.erl +++ b/lib/snmp/test/snmp_test_mgr_misc.erl @@ -576,6 +576,7 @@ vsn('version-2') -> v2c. udp_send(UdpId, AgentIp, UdpPort, B) -> + ?vlog("attempt send message (~w bytes) to ~p", [sz(B), {AgentIp, UdpPort}]), case (catch gen_udp:send(UdpId, AgentIp, UdpPort, B)) of {error,ErrorReason} -> error("failed (error) sending message to ~p:~p: " @@ -880,7 +881,7 @@ display_prop_hdr(S) -> %%---------------------------------------------------------------------- sz(L) when is_list(L) -> - length(lists:flatten(L)); + iolist_size(L); sz(B) when is_binary(B) -> size(B); sz(O) -> diff --git a/lib/snmp/test/snmp_to_snmpnet_SUITE.erl b/lib/snmp/test/snmp_to_snmpnet_SUITE.erl index b83c7461d1..b8bdb30271 100644 --- a/lib/snmp/test/snmp_to_snmpnet_SUITE.erl +++ b/lib/snmp/test/snmp_to_snmpnet_SUITE.erl @@ -247,9 +247,17 @@ erlang_agent_netsnmp_get(Config) when is_list(Config) -> start_agent(Config), Oid = ?sysDescr_instance, Expected = expected(Oid, get), - [Expected = snmpget(Oid, Transport, Config) - || Transport <- Transports], - ok. + try + begin + [Expected = snmpget(Oid, Transport, Config) + || Transport <- Transports], + ok + end + catch + throw:{skip, _} = SKIP -> + SKIP + end. + %%-------------------------------------------------------------------- erlang_manager_netsnmp_get() -> @@ -260,29 +268,34 @@ erlang_manager_netsnmp_get(Config) when is_list(Config) -> SysDescr = "Net-SNMP agent", TargetName = "Target Net-SNMP agent", Transports = ?config(transports, Config), - ProgHandle = start_snmpd(Community, SysDescr, Config), - start_manager(Config), - snmp_manager_user:start_link(self(), test_user), - [snmp_manager_user:register_agent( - TargetName++domain_suffix(Domain), - [{reg_type, target_name}, - {tdomain, Domain}, {taddress, Addr}, - {community, Community}, {engine_id, "EngineId"}, - {version, v2}, {sec_model, v2c}, {sec_level, noAuthNoPriv}]) - || {Domain, Addr} <- Transports], - Results = - [snmp_manager_user:sync_get( - TargetName++domain_suffix(Domain), - [?sysDescr_instance]) - || {Domain, _} <- Transports], - ct:pal("sync_get -> ~p", [Results]), - snmp_manager_user:stop(), - stop_program(ProgHandle), - [{ok, - {noError, 0, - [{varbind, ?sysDescr_instance, 'OCTET STRING', SysDescr,1}] }, - _} = R || R <- Results], - ok. + case start_snmpd(Community, SysDescr, Config) of + {skip, _} = SKIP -> + SKIP; + ProgHandle -> + start_manager(Config), + snmp_manager_user:start_link(self(), test_user), + [snmp_manager_user:register_agent( + TargetName++domain_suffix(Domain), + [{reg_type, target_name}, + {tdomain, Domain}, {taddress, Addr}, + {community, Community}, {engine_id, "EngineId"}, + {version, v2}, {sec_model, v2c}, {sec_level, noAuthNoPriv}]) + || {Domain, Addr} <- Transports], + Results = + [snmp_manager_user:sync_get( + TargetName++domain_suffix(Domain), + [?sysDescr_instance]) + || {Domain, _} <- Transports], + ct:pal("sync_get -> ~p", [Results]), + snmp_manager_user:stop(), + stop_program(ProgHandle), + [{ok, + {noError, 0, + [{varbind, ?sysDescr_instance, 'OCTET STRING', SysDescr,1}] }, + _} = R || R <- Results], + ok + end. + %%-------------------------------------------------------------------- erlang_agent_netsnmp_inform(Config) when is_list(Config) -> @@ -292,17 +305,19 @@ erlang_agent_netsnmp_inform(Config) when is_list(Config) -> start_agent(Config), ok = snmpa:load_mib(snmp_master_agent, filename:join(DataDir, Mib)), - ProgHandle = start_snmptrapd(Mib, Config), - - snmpa:send_notification( - snmp_master_agent, testTrapv22, {erlang_agent_test, self()}), - - receive - {snmp_targets, erlang_agent_test, Addresses} -> - ct:pal("Notification sent to: ~p~n", [Addresses]), - erlang_agent_netsnmp_inform_responses(Addresses) - end, - stop_program(ProgHandle). + case start_snmptrapd(Mib, Config) of + {skip, _} = SKIP -> + SKIP; + ProgHandle -> + snmpa:send_notification( + snmp_master_agent, testTrapv22, {erlang_agent_test, self()}), + receive + {snmp_targets, erlang_agent_test, Addresses} -> + ct:pal("Notification sent to: ~p~n", [Addresses]), + erlang_agent_netsnmp_inform_responses(Addresses) + end, + stop_program(ProgHandle) + end. erlang_agent_netsnmp_inform_responses([]) -> receive @@ -326,6 +341,7 @@ erlang_agent_netsnmp_inform_responses([Address | Addresses]) -> %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- + snmpget(Oid, Transport, Config) -> Versions = ?config(snmp_versions, Config), @@ -335,10 +351,14 @@ snmpget(Oid, Transport, Config) -> "-Cf", net_snmp_addr_str(Transport), oid_str(Oid)], - ProgHandle = start_program(snmpget, Args, none, Config), - {_, line, Line} = get_program_output(ProgHandle), - stop_program(ProgHandle), - Line. + case start_program(snmpget, Args, none, Config) of + {skip, _} = SKIP -> + throw(SKIP); + ProgHandle -> + {_, line, Line} = get_program_output(ProgHandle), + stop_program(ProgHandle), + Line + end. start_snmptrapd(Mibs, Config) -> DataDir = ?config(data_dir, Config), @@ -382,12 +402,13 @@ start_program(Prog, Args, StartCheckMP, Config) -> DataDir = ?config(data_dir, Config), StartWrapper = filename:join(DataDir, "start_stop_wrapper"), Parent = self(), - Pid = - spawn_link( + %% process_flag(trap_exit, true), + {Pid, Mon} = + spawn_monitor( fun () -> run_program(Parent, StartWrapper, [Path | Args]) end), - start_check(Pid, erlang:monitor(process, Pid), StartCheckMP). + start_check(Pid, Mon, StartCheckMP). start_check(Pid, Mon, none) -> {Pid, Mon}; @@ -400,6 +421,10 @@ start_check(Pid, Mon, StartCheckMP) -> nomatch -> start_check(Pid, Mon, StartCheckMP) end; + {'DOWN', Mon, _, _Pid, {skip, Reason} = SKIP} -> + ct:pal("Received DOWN from ~p" + "~n Skip Reason: ~p", [_Pid, Reason]), + SKIP; {'DOWN', Mon, _, _, Reason} -> ct:fail("Prog ~p start failed: ~p", [Pid, Reason]) end. @@ -446,14 +471,34 @@ run_program_loop(Parent, Port, Buf) -> {Port, {data, {Flag, Data}}} -> case Flag of eol -> - Line = iolist_to_binary(lists:reverse(Buf, Data)), - ct:pal("Prog ~p output: ~s", [Port, Line]), - Parent ! {self(), line, Line}, - run_program_loop(Parent, Port, []); + Line = iolist_to_binary(lists:reverse(Buf, Data)), + ct:pal("Prog ~p output: ~s", [Port, Line]), + %% There are potentially many different fail outputs, + %% but for now we test for just this one: illegal option + IOpt = "illegal option", + case string:find(binary_to_list(Line), IOpt) of + nomatch -> + Parent ! {self(), line, Line}, + run_program_loop(Parent, Port, []); + Line2 -> + %% Try to extract the actual illegal option string + IOpt2 = + case string:take( + string:prefix(Line2, IOpt), [$-, $ ]) of + {_, Str} when length(Str) > 0 -> + Str; + _X -> + Line2 + end, + ct:pal("Force program ~p stop", [Port]), + true = port_command(Port, <<"stop\n">>), + (catch port_close(Port)), + exit({skip, {illegal_option, IOpt2}}) + end; noeol -> run_program_loop(Parent, Port, [Data | Buf]) end; - {Port, {exit_status,ExitStatus}} -> + {Port, {exit_status, ExitStatus}} -> ct:pal("Prog ~p exit: ~p", [Port, ExitStatus]), catch port_close(Port), Parent ! {self(), exit, ExitStatus}; diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 4de51c9a35..a0ae51ed0a 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -1180,7 +1180,7 @@ validate_certificate_chain(Certs, CertDbHandle, CertDbRef, SslOptions, CRLDbHand CertDbHandle, CertDbRef)} end catch - error:{badmatch,{asn1, Asn1Reason}} -> + error:{badmatch,{error, {asn1, Asn1Reason}}} -> %% ASN-1 decode of certificate somehow failed {error, {certificate_unknown, {failed_to_decode_certificate, Asn1Reason}}}; error:OtherReason -> diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 726b409d4d..197564b895 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -33,6 +33,8 @@ %%% BIFs +-export([internal_run/4]). + -export([version/0, compile/1, compile/2, run/2, run/3, inspect/2]). -spec version() -> binary(). @@ -100,6 +102,40 @@ run(_, _) -> run(_, _, _) -> erlang:nif_error(undef). +-spec internal_run(Subject, RE, Options, FirstCall) -> {match, Captured} | + match | + nomatch | + {error, ErrType} when + Subject :: iodata() | unicode:charlist(), + RE :: mp() | iodata() | unicode:charlist(), + Options :: [Option], + Option :: anchored | global | notbol | noteol | notempty + | notempty_atstart | report_errors + | {offset, non_neg_integer()} | + {match_limit, non_neg_integer()} | + {match_limit_recursion, non_neg_integer()} | + {newline, NLSpec :: nl_spec()} | + bsr_anycrlf | bsr_unicode | {capture, ValueSpec} | + {capture, ValueSpec, Type} | CompileOpt, + Type :: index | list | binary, + ValueSpec :: all | all_but_first | all_names | first | none | ValueList, + ValueList :: [ValueID], + ValueID :: integer() | string() | atom(), + CompileOpt :: compile_option(), + Captured :: [CaptureData] | [[CaptureData]], + CaptureData :: {integer(), integer()} + | ListConversionData + | binary(), + ListConversionData :: string() + | {error, string(), binary()} + | {incomplete, string(), binary()}, + ErrType :: match_limit | match_limit_recursion | {compile, CompileErr}, + CompileErr :: {ErrString :: string(), Position :: non_neg_integer()}, + FirstCall :: boolean(). + +internal_run(_, _, _, _) -> + erlang:nif_error(undef). + -spec inspect(MP,Item) -> {namelist, [ binary() ]} when MP :: mp(), Item :: namelist. @@ -765,17 +801,17 @@ do_grun(FlatSubject,Subject,Unicode,CRLF,RE,{Options0,NeedClean}) -> try postprocess(loopexec(FlatSubject,RE,InitialOffset, byte_size(FlatSubject), - Unicode,CRLF,StrippedOptions), + Unicode,CRLF,StrippedOptions,true), SelectReturn,ConvertReturn,FlatSubject,Unicode) catch throw:ErrTuple -> ErrTuple end. -loopexec(_,_,X,Y,_,_,_) when X > Y -> +loopexec(_,_,X,Y,_,_,_,_) when X > Y -> {match,[]}; -loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> - case re:run(Subject,RE,[{offset,X}]++Options) of +loopexec(Subject,RE,X,Y,Unicode,CRLF,Options, First) -> + case re:internal_run(Subject,RE,[{offset,X}]++Options,First) of {error, Err} -> throw({error,Err}); nomatch -> @@ -784,11 +820,11 @@ loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> {match,Rest} = case B>0 of true -> - loopexec(Subject,RE,A+B,Y,Unicode,CRLF,Options); + loopexec(Subject,RE,A+B,Y,Unicode,CRLF,Options,false); false -> {match,M} = - case re:run(Subject,RE,[{offset,X},notempty_atstart, - anchored]++Options) of + case re:internal_run(Subject,RE,[{offset,X},notempty_atstart, + anchored]++Options,false) of nomatch -> {match,[]}; {match,Other} -> @@ -801,7 +837,7 @@ loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> forward(Subject,A,1,Unicode,CRLF) end, {match,MM} = loopexec(Subject,RE,NewA,Y, - Unicode,CRLF,Options), + Unicode,CRLF,Options,false), case M of [] -> {match,MM}; diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index ecb514e9f3..d7d57941c2 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -108,7 +108,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-10.4","crypto-3.3", + {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-@OTP-15831:OTP-15836@","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index c9ef9da990..06d8fe9255 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -28,7 +28,8 @@ pcre_compile_workspace_overflow/1,re_infinite_loop/1, re_backwards_accented/1,opt_dupnames/1,opt_all_names/1,inspect/1, opt_no_start_optimize/1,opt_never_utf/1,opt_ucp/1, - match_limit/1,sub_binaries/1,copt/1]). + match_limit/1,sub_binaries/1,copt/1,global_unicode_validation/1, + yield_on_subject_validation/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -45,7 +46,8 @@ all() -> pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, inspect, opt_no_start_optimize,opt_never_utf,opt_ucp, - match_limit, sub_binaries, re_version]. + match_limit, sub_binaries, re_version, global_unicode_validation, + yield_on_subject_validation]. groups() -> []. @@ -200,7 +202,58 @@ re_version(_Config) -> {match,[Version]} = re:run(Version,"^[0-9]\\.[0-9]{2} 20[0-9]{2}-[0-9]{2}-[0-9]{2}",[{capture,all,binary}]), ok. +global_unicode_validation(Config) when is_list(Config) -> + %% Test that unicode validation of the subject is not done + %% for every match found... + Bin = binary:copy(<<"abc\n">>,100000), + {TimeAscii, _} = take_time(fun () -> + re:run(Bin, <<"b">>, [global]) + end), + {TimeUnicode, _} = take_time(fun () -> + re:run(Bin, <<"b">>, [unicode,global]) + end), + if TimeAscii == 0; TimeUnicode == 0 -> + {comment, "Not good enough resolution to compare results"}; + true -> + %% The time the operations takes should be in the + %% same order of magnitude. If validation of the + %% whole subject occurs for every match, the unicode + %% variant will take way longer time... + true = TimeUnicode div TimeAscii < 10 + end. + +take_time(Fun) -> + Start = erlang:monotonic_time(nanosecond), + Res = Fun(), + End = erlang:monotonic_time(nanosecond), + {End-Start, Res}. + +yield_on_subject_validation(Config) when is_list(Config) -> + Go = make_ref(), + Bin = binary:copy(<<"abc\n">>,100000), + {P, M} = spawn_opt(fun () -> + receive Go -> ok end, + {match,[{1,1}]} = re:run(Bin, <<"b">>, [unicode]) + end, + [link, monitor]), + 1 = erlang:trace(P, true, [running]), + P ! Go, + N = count_re_run_trap_out(P, M), + true = N >= 5, + ok. +count_re_run_trap_out(P, M) when is_reference(M) -> + receive {'DOWN',M,process,P,normal} -> ok end, + TD = erlang:trace_delivered(P), + receive {trace_delivered, P, TD} -> ok end, + count_re_run_trap_out(P, 0); +count_re_run_trap_out(P, N) when is_integer(N) -> + receive + {trace,P,out,{erlang,re_run_trap,3}} -> + count_re_run_trap_out(P, N+1) + after 0 -> + N + end. %% Test compile options given directly to run. combined_options(Config) when is_list(Config) -> diff --git a/system/doc/reference_manual/typespec.xml b/system/doc/reference_manual/typespec.xml index 27cd0ba83d..f517259a64 100644 --- a/system/doc/reference_manual/typespec.xml +++ b/system/doc/reference_manual/typespec.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2018</year> + <year>2003</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -140,8 +140,7 @@ | nonempty_improper_list(Type1, Type2) %% Type1 and Type2 as above | nonempty_list(Type) %% Proper non-empty list - Map :: map() %% denotes a map of any size - | #{} %% denotes the empty map + Map :: #{} %% denotes the empty map | #{AssociationList} Tuple :: tuple() %% denotes a tuple of any size @@ -192,17 +191,16 @@ <c>AssociationList</c> are allowed to overlap, and if they do, the leftmost association takes precedence. A map association has a key in <c>AssociationList</c> if it belongs to this type. - <c>AssociationList</c> can contain both mandatory and optional - association types. + <c>AssociationList</c> can contain both mandatory <c>(:=)</c> and + optional <c>(=>)</c> association types. If an association type is mandatory, an association with that type - is to be present. + needs to be present. In the case of an optional association type it is not required for the key type to be present. </p> <p> - Notice that the syntactic representation of <c>map()</c> is - <c>#{any() => any()}</c> (or <c>#{_ => _}</c>), not <c>#{}</c>. The notation <c>#{}</c> specifies the singleton type for the empty map. + Note that this notation is not a shorthand for the <c>map()</c> type. </p> <p> For convenience, the following types are also built-in. @@ -259,6 +257,9 @@ <cell><c>iolist()</c></cell><cell><c>maybe_improper_list(byte() | binary() | iolist(), binary() | [])</c></cell> </row> <row> + <cell><c>map()</c></cell><cell><c>#{any() => any()}</c></cell> + </row> + <row> <cell><c>function()</c></cell><cell><c>fun()</c></cell> </row> <row> |