diff options
48 files changed, 1543 insertions, 491 deletions
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 941c3ebbbe..35f2ea6688 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4612,7 +4612,15 @@ typedef struct SortGenOpArg { static int genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b) { - return CMP_TERM(a->term, b->term); + Sint res = CMP_TERM(a->term, b->term); + + if (res < 0) { + return -1; + } else if (res > 0) { + return 1; + } + + return 0; } static int diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index db9c258cb7..602db106b1 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -413,6 +413,7 @@ bif re:compile/1 bif re:compile/2 bif re:run/2 bif re:run/3 +bif re:internal_run/4 # # Bifs in lists module. diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d8501ea6ac..4537e3e569 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -80,7 +80,7 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) byte *extp = edep->data->extp; Eterm msg; Sint ctl_len; - Sint size = ctl_len = erts_decode_dist_ext_size(edep, 0); + Sint size = ctl_len = erts_decode_dist_ext_size(edep, 0, 0); if (size < 0) { erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -1462,7 +1462,7 @@ int erts_net_message(Port *prt, #endif goto data_error; case ERTS_PREP_DIST_EXT_SUCCESS: - ctl_len = erts_decode_dist_ext_size(&ede, 1); + ctl_len = erts_decode_dist_ext_size(&ede, 1, 0); if (ctl_len < 0) { #ifdef ERTS_DIST_MSG_DBG erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); @@ -1543,39 +1543,6 @@ int erts_net_message(Port *prt, edep = erts_get_dist_ext(&seq->hfrag); ede_hfrag = &seq->hfrag; - /* If the sequence consisted of more than 1 fragment we create one large - binary out of all of the fragments. This because erts_decode_ext - cannot handle a segmented buffer. - TODO: Move this copy to as late as possible, preferably in in the - erts_decode_dist_ext in the receiving process. - */ - if (edep->data->frag_id > 1) { - Uint sz = 0; - Binary *bin; - int i; - byte *ep; - - for (i = 0; i < edep->data->frag_id; i++) - sz += edep->data[i].ext_endp - edep->data[i].extp; - - bin = erts_bin_nrml_alloc(sz); - ep = (byte*)bin->orig_bytes; - - for (i = 0; i < edep->data->frag_id; i++) { - sys_memcpy(ep, edep->data[i].extp, edep->data[i].ext_endp - edep->data[i].extp); - ep += edep->data[i].ext_endp - edep->data[i].extp; - erts_bin_release(edep->data[i].binp); - edep->data[i].binp = NULL; - edep->data[i].extp = NULL; - edep->data[i].ext_endp = NULL; - } - - edep->data->frag_id = 1; - edep->data->extp = (byte*)bin->orig_bytes; - edep->data->ext_endp = ep; - edep->data->binp = bin; - } - break; } default: diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index b23fa77f5f..fa2edfef1e 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -413,12 +413,25 @@ typedef struct { #define ERTS_RBT_GET_LEFT(T) ((T)->left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) #define ERTS_RBT_GET_KEY(T) ((T)->key) -#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY) +#define ERTS_RBT_CMP_KEYS(KX, KY) subtract_term_cmp((KX), (KY)) #define ERTS_RBT_WANT_LOOKUP_INSERT #define ERTS_RBT_WANT_LOOKUP #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_UNDEF +/* erl_rbtree expects comparisons to return an int */ +static int subtract_term_cmp(Eterm a, Eterm b) { + Sint res = CMP_TERM(a, b); + + if (res < 0) { + return -1; + } else if (res > 0) { + return 1; + } + + return 0; +} + #include "erl_rbtree.h" static int subtract_continue(Process *p, ErtsSubtractContext *context); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index e0b9202fe7..b3bf1c7ee3 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -46,7 +46,7 @@ static Export *urun_trap_exportp = NULL; static Export *ucompile_trap_exportp = NULL; static BIF_RETTYPE re_exec_trap(BIF_ALIST_3); -static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first); static void *erts_erts_pcre_malloc(size_t size) { return erts_alloc(ERTS_ALC_T_RE_HEAP,size); @@ -1094,7 +1094,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) * The actual re:run/2,3 BIFs */ static BIF_RETTYPE -re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first) { const pcre *code_tmp; RestartContext restart; @@ -1120,6 +1120,14 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) < 0) { BIF_ERROR(p,BADARG); } + if (!first) { + /* + * 'first' is false when re:grun() previously has called re:internal_run() + * with the same subject; i.e., no need to do yet another validation of + * the subject regarding utf8 encoding... + */ + options |= PCRE_NO_UTF8_CHECK; + } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); @@ -1360,15 +1368,28 @@ handle_iolist: } BIF_RETTYPE +re_internal_run_4(BIF_ALIST_4) +{ + int first; + if (BIF_ARG_4 == am_false) + first = 0; + else if (BIF_ARG_4 == am_true) + first = !0; + else + BIF_ERROR(BIF_P,BADARG); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, first); +} + +BIF_RETTYPE re_run_3(BIF_ALIST_3) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, !0); } BIF_RETTYPE re_run_2(BIF_ALIST_2) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL, !0); } /* @@ -1407,6 +1428,7 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3) loop_count = 0xFFFFFFFF; #endif rc = erts_pcre_exec(NULL, &(restartp->extra), NULL, 0, 0, 0, NULL, 0); + ASSERT(loop_count != 0xFFFFFFFF); BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 6645341512..1bebf6efe2 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -527,7 +527,7 @@ erts_msg_attached_data_size_aux(ErtsMessage *msg) if (edep->heap_size < 0) { - sz = erts_decode_dist_ext_size(edep, 1); + sz = erts_decode_dist_ext_size(edep, 1, 1); if (sz < 0) { /* Bad external * We leave the message intact in this case as it's not worth the trouble diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index 55e469b553..fb900ca7ba 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -3028,7 +3028,7 @@ erts_proc_sig_decode_dist(Process *proc, ErtsProcLocks proc_locks, if (edep->heap_size >= 0) need = edep->heap_size; else { - need = erts_decode_dist_ext_size(edep, 1); + need = erts_decode_dist_ext_size(edep, 1, 1); if (need < 0) { /* bad signal; remove it... */ return 0; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 1f6adb98ef..de0564292d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8568,9 +8568,6 @@ erts_start_schedulers(void) { ethr_tid tid; int res = 0; - Uint actual; - Uint wanted = erts_no_schedulers; - Uint wanted_no_schedulers = erts_no_schedulers; char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; int ix; @@ -8584,40 +8581,34 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) - erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); + erts_exit(ERTS_ABORT_EXIT, "Failed to create run-queue supervision event\n"); res = ethr_thr_create(&runq_supervisor_tid, runq_supervisor, NULL, &opts); if (0 != res) - erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread, " + erts_exit(ERTS_ABORT_EXIT, "Failed to create run-queue supervision thread, " "error = %d\n", res); } opts.suggested_stack_size = erts_sched_thread_suggested_stack_size; - if (wanted < 1) - wanted = 1; - if (wanted > ERTS_MAX_NO_OF_SCHEDULERS) { - wanted = ERTS_MAX_NO_OF_SCHEDULERS; - res = ENOTSUP; - } - - for (actual = 0; actual < wanted; actual++) { - ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); - - ASSERT(actual == esdp->no - 1); - - erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); + ASSERT(erts_no_schedulers > 0 && erts_no_schedulers <= ERTS_MAX_NO_OF_SCHEDULERS); + for (ix = 0; ix < erts_no_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); + ASSERT(ix == esdp->no - 1); + erts_snprintf(opts.name, 16, "%lu_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); - if (res != 0) { - break; + erts_exit(ERTS_ABORT_EXIT, "Failed to create scheduler thread %d, error = %d\n", ix, res); } } - erts_no_schedulers = actual; + + /* Probably not needed as thread create will imply a memory barrier, + but we do one just to be safe. */ + ERTS_THR_MEMORY_BARRIER; { for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { @@ -8626,7 +8617,7 @@ erts_start_schedulers(void) opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res); + erts_exit(ERTS_ABORT_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); @@ -8634,40 +8625,22 @@ erts_start_schedulers(void) opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res); + erts_exit(ERTS_ABORT_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res); } } - ERTS_THR_MEMORY_BARRIER; - erts_snprintf(opts.name, 16, "aux"); res = ethr_thr_create(&tid, aux_thread, NULL, &opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread, error = %d\n", res); + erts_exit(ERTS_ABORT_EXIT, "Failed to create aux thread, error = %d\n", res); for (ix = 0; ix < erts_no_poll_threads; ix++) { erts_snprintf(opts.name, 16, "%d_poller", ix); res = ethr_thr_create(&tid, poll_thread, (void*)(UWord)ix, &opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create poll thread\n"); - } - - if (actual < 1) - erts_exit(ERTS_ERROR_EXIT, - "Failed to create any scheduler-threads: %s (%d)\n", - erl_errno_id(res), - res); - if (res != 0) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - ASSERT(actual != wanted_no_schedulers); - erts_dsprintf(dsbufp, - "Failed to create %beu scheduler-threads (%s:%d); " - "only %beu scheduler-thread%s created.\n", - wanted_no_schedulers, erl_errno_id(res), res, - actual, actual == 1 ? " was" : "s were"); - erts_send_error_to_logger_nogl(dsbufp); + erts_exit(ERTS_ABORT_EXIT, "Failed to create poll thread\n"); } } @@ -12097,6 +12070,7 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) ErtsHeapFactory factory; Sint reds_consumed = 0; + ASSERT(c_p->flags & F_DISABLE_GC); ASSERT(erts_monitor_is_target(mon) && mon->type == ERTS_MON_TYPE_DIST_PROC); mdp = erts_monitor_to_data(mon); @@ -12144,7 +12118,6 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) switch (code) { case ERTS_DSIG_SEND_CONTINUE: case ERTS_DSIG_SEND_YIELD: - erts_set_gc_state(c_p, 0); ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); reds_consumed = reds; /* force yield */ break; @@ -12152,7 +12125,6 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) break; case ERTS_DSIG_SEND_TOO_LRG: erts_kill_dist_connection(dep, dist->connection_id); - erts_set_gc_state(c_p, 1); break; default: ASSERT(! "Invalid dsig send exit monitor result"); @@ -12356,6 +12328,7 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) ErtsHeapFactory factory; Sint reds_consumed = 0; + ASSERT(c_p->flags & F_DISABLE_GC); ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); dlnk = erts_link_to_other(lnk, &ldp); dist = ((ErtsLinkDataExtended *) ldp)->dist; @@ -12395,7 +12368,6 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) switch (code) { case ERTS_DSIG_SEND_YIELD: case ERTS_DSIG_SEND_CONTINUE: - erts_set_gc_state(c_p, 0); ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); reds_consumed = reds; /* force yield */ break; @@ -12403,7 +12375,6 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) break; case ERTS_DSIG_SEND_TOO_LRG: erts_kill_dist_connection(dep, dist->connection_id); - erts_set_gc_state(c_p, 1); break; default: ASSERT(! "Invalid dsig send exit monitor result"); @@ -12951,6 +12922,8 @@ restart: yield_allowed = 0; #endif + /* Enable GC again, through strictly not needed it puts + the process in a consistent state. */ erts_set_gc_state(p, 1); /* Set state to not active as we don't want this process diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 0d6b512f78..745a2a482c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -2353,6 +2353,8 @@ erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq) old_rqint); if (act_rqint == old_rqint) return !0; + + old_rqint = act_rqint; } } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index ec67ab2aed..ce61cdf040 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1062,11 +1062,38 @@ bad_dist_ext(ErtsDistExternal *edep) } Sint -erts_decode_dist_ext_size(ErtsDistExternal *edep, int kill_connection) +erts_decode_dist_ext_size(ErtsDistExternal *edep, int kill_connection, int payload) { Sint res; byte *ep; + if (edep->data->frag_id > 1 && payload) { + Uint sz = 0; + Binary *bin; + int i; + byte *ep; + + for (i = 0; i < edep->data->frag_id; i++) + sz += edep->data[i].ext_endp - edep->data[i].extp; + + bin = erts_bin_nrml_alloc(sz); + ep = (byte*)bin->orig_bytes; + + for (i = 0; i < edep->data->frag_id; i++) { + sys_memcpy(ep, edep->data[i].extp, edep->data[i].ext_endp - edep->data[i].extp); + ep += edep->data[i].ext_endp - edep->data[i].extp; + erts_bin_release(edep->data[i].binp); + edep->data[i].binp = NULL; + edep->data[i].extp = NULL; + edep->data[i].ext_endp = NULL; + } + + edep->data->frag_id = 1; + edep->data->extp = (byte*)bin->orig_bytes; + edep->data->ext_endp = ep; + edep->data->binp = bin; + } + if (edep->data->extp >= edep->data->ext_endp) goto fail; #ifndef ERTS_DEBUG_USE_DIST_SEP @@ -1164,6 +1191,7 @@ Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags) if (flags) { ASSERT(flags == ERTS_DIST_EXT_BTT_SAFE); ede.flags = flags; /* a dummy struct just for the flags */ + ede.data = NULL; edep = &ede; } else { edep = NULL; @@ -1233,8 +1261,10 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) ede.data->extp = binary_bytes(real_bin)+offset; ede.data->ext_endp = ede.data->extp + size; + ede.data->frag_id = 1; + ede.data->binp = NULL; - hsz = erts_decode_dist_ext_size(&ede, 1); + hsz = erts_decode_dist_ext_size(&ede, 1, 1); if (hsz < 0) goto badarg; @@ -1765,6 +1795,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx) case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; + fakedep.data = NULL; dec_term(&fakedep, NULL, NULL, NULL, ctx); break; } @@ -3762,6 +3793,30 @@ dec_term_atom_common: hp += heap_bin_size(n); sys_memcpy(hb->data, ep, n); *objp = make_binary(hb); + } else if (edep && edep->data && edep->data->binp && + n > (edep->data->binp->orig_size / 4)) { + /* If we decode a refc binary from a distribution data + entry we know that it is a refc binary to begin with + so we just increment it and use the reference. This + means that the entire distribution data entry will + remain until this binary is de-allocated so we only + do it if a substantial part (> 25%) of the data + is a binary. */ + ProcBin* pb = (ProcBin *) hp; + Binary* bptr = edep->data->binp; + erts_refc_inc(&bptr->intern.refc, 1); + pb->thing_word = HEADER_PROC_BIN; + pb->size = n; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + pb->val = bptr; + pb->bytes = (byte*) ep; + ERTS_ASSERT((byte*)(bptr->orig_bytes) < ep && + ep+n <= (byte*)(bptr->orig_bytes+bptr->orig_size)); + pb->flags = 0; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); + hp += PROC_BIN_SIZE; + *objp = make_binary(pb); } else { Binary* dbin = erts_bin_nrml_alloc(n); diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index b556c9076c..e362a6c81f 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -199,7 +199,7 @@ typedef enum { ErtsPrepDistExtRes erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, struct binary *, DistEntry *, Uint32, ErtsAtomCache *); -Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); +Sint erts_decode_dist_ext_size(ErtsDistExternal *, int, int); Eterm erts_decode_dist_ext(ErtsHeapFactory*, ErtsDistExternal *, int); Sint erts_decode_ext_size(byte*, Uint); diff --git a/erts/emulator/nifs/common/socket_nif.c b/erts/emulator/nifs/common/socket_nif.c index adecbb3b6e..56a16a87a1 100644 --- a/erts/emulator/nifs/common/socket_nif.c +++ b/erts/emulator/nifs/common/socket_nif.c @@ -9052,35 +9052,67 @@ ERL_NIF_TERM nsetopt_lvl_ip_update_membership(ErlNifEnv* env, #endif // It must be a map - if (!IS_MAP(env, eVal)) + if (!IS_MAP(env, eVal)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "value *not* a map\r\n") ); return enif_make_badarg(env); + } // It must have atleast two attributes - if (!enif_get_map_size(env, eVal, &sz) || (sz >= 2)) + if (!enif_get_map_size(env, eVal, &sz) || (sz < 2)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "invalid map value: %T\r\n", eVal) ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) + if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed get multiaddr (map) attribute\r\n") ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) + if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed get interface (map) attribute\r\n") ); return enif_make_badarg(env); + } if ((xres = esock_decode_ip4_address(env, eMultiAddr, - &mreq.imr_multiaddr)) != NULL) + &mreq.imr_multiaddr)) != NULL) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed decode multiaddr %T: %s\r\n", eMultiAddr, xres) ); return esock_make_error_str(env, xres); + } if ((xres = esock_decode_ip4_address(env, eInterface, - &mreq.imr_interface)) != NULL) + &mreq.imr_interface)) != NULL) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed decode interface %T: %s\r\n", eInterface, xres) ); return esock_make_error_str(env, xres); + } res = socket_setopt(descP->sock, level, opt, &mreq, sizeof(mreq)); - if (res != 0) - result = esock_make_error_errno(env, sock_errno()); - else + if (res != 0) { + int save_errno = sock_errno(); + + result = esock_make_error_errno(env, save_errno); + + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed setopt: %T (%d)\r\n", result, save_errno) ); + + } else { result = esock_atom_ok; + } return result; } @@ -9684,33 +9716,65 @@ ERL_NIF_TERM nsetopt_lvl_ipv6_update_membership(ErlNifEnv* env, #endif // It must be a map - if (!IS_MAP(env, eVal)) + if (!IS_MAP(env, eVal)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "value *not* a map\r\n") ); return enif_make_badarg(env); + } // It must have atleast two attributes - if (!enif_get_map_size(env, eVal, &sz) || (sz >= 2)) + if (!enif_get_map_size(env, eVal, &sz) || (sz < 2)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "invalid map value: %T\r\n", eVal) ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) + if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed get multiaddr (map) attribute\r\n") ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) + if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed get interface (map) attribute\r\n") ); return enif_make_badarg(env); + } if ((xres = esock_decode_ip6_address(env, eMultiAddr, - &mreq.ipv6mr_multiaddr)) != NULL) + &mreq.ipv6mr_multiaddr)) != NULL) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed decode multiaddr %T: %s\r\n", eMultiAddr, xres) ); return esock_make_error_str(env, xres); + } - if (!GET_UINT(env, eInterface, &mreq.ipv6mr_interface)) + if (!GET_UINT(env, eInterface, &mreq.ipv6mr_interface)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed decode interface %T: %s\r\n", eInterface, xres) ); return esock_make_error(env, esock_atom_einval); + } res = socket_setopt(descP->sock, level, opt, &mreq, sizeof(mreq)); - if (res != 0) - result = esock_make_error_errno(env, sock_errno()); - else + if (res != 0) { + int save_errno = sock_errno(); + + result = esock_make_error_errno(env, save_errno); + + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed setopt: %T (%d)\r\n", result, save_errno) ); + + } else { result = esock_atom_ok; + } return result; } diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h index 3563791223..505e2ccce0 100644 --- a/erts/emulator/pcre/pcre.h +++ b/erts/emulator/pcre/pcre.h @@ -240,6 +240,9 @@ with J. */ #define PCRE_UTF8_ERR20 20 #define PCRE_UTF8_ERR21 21 #define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ +#if defined(ERLANG_INTEGRATION) +#define PCRE_UTF8_YIELD 23 +#endif /* Specific error codes for UTF-16 validity checks */ diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 1946e97a72..55a7b377bf 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -6642,10 +6642,16 @@ typedef struct { REAL_PCRE *Xre; heapframe Xframe_zero; /* Always NO_RECURSE */ + /* for yield in valid_utf() */ + + struct PRIV(valid_utf_ystate) valid_utf_ystate; + /* Original function parameters that need be saved */ int Xstart_offset; int Xoffsetcount; int *Xoffsets; + int Xlength; + PCRE_SPTR Xsubject; } PcreExecContext; #endif @@ -6675,6 +6681,7 @@ pcre32_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, #endif { #ifndef ERLANG_INTEGRATION +#define ERTS_UPDATE_CONSUMED(X, MD) int rc, ocount, arg_offset_max; int newline; BOOL using_temporary_offsets = FALSE; @@ -6736,6 +6743,8 @@ heapframe frame_zero; start_offset = exec_context->Xstart_offset; \ offsetcount = exec_context->Xoffsetcount; \ offsets = exec_context->Xoffsets; \ + length = exec_context->Xlength; \ + subject = exec_context->Xsubject; \ } while (0) #define SWAPOUT() do { \ @@ -6750,8 +6759,30 @@ heapframe frame_zero; exec_context->Xstart_offset = start_offset; \ exec_context->Xoffsetcount = offsetcount; \ exec_context->Xoffsets = offsets; \ + exec_context->Xlength = length; \ + exec_context->Xsubject = subject; \ } while (0) +#define ERTS_UPDATE_CONSUMED(X, MD) \ +do { \ + if (((X)->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { \ + unsigned long consumed__; \ + if (!(X)->restart_data) { \ + consumed__ = 0; \ + } \ + else { \ + PcreExecContext *ctx__ = (PcreExecContext *) \ + (*(X)->restart_data); \ + consumed__ = ctx__->valid_utf_ystate.cnt; \ + ctx__->valid_utf_ystate.cnt = 0; \ + } \ + if ((MD)) { \ + match_data *md__ = (MD); \ + consumed__ += (X)->loop_limit - md__->loop_limit; \ + } \ + *((X)->loop_counter_return) = consumed__; \ + } \ +} while (0) PcreExecContext *exec_context; PcreExecContext internal_context; @@ -6776,15 +6807,21 @@ pcre_uchar req_char; /* we are restarting, every initialization is skipped and we jump directly into the loop */ exec_context = (PcreExecContext *) *(extra_data->restart_data); SWAPIN(); - + if (exec_context->valid_utf_ystate.yielded) + goto restart_valid_utf; goto RESTART_INTERRUPTED; } else { if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { exec_context = (PcreExecContext *) (erts_pcre_malloc)(sizeof(PcreExecContext)); - *(extra_data->restart_data) = (void *) exec_context; + *(extra_data->restart_data) = (void *) exec_context; + exec_context->valid_utf_ystate.yielded = 0; /* need freeing by special routine from client */ } else { +#if defined(ERLANG_INTEGRATION) + fprintf(stderr, "Unexpected execution path\n"); + abort(); +#endif exec_context = &internal_context; } @@ -6865,9 +6902,38 @@ code for an invalid string if a results vector is available. */ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) { int erroroffset; - int errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length, &erroroffset); + int errorcode; + +#if !defined(ERLANG_INTEGRATION) + errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length); +#else + struct PRIV(valid_utf_ystate) *ystate; + + if (!extra_data || !extra_data->restart_data) { + ystate = NULL; + } + else if (!(extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { + exec_context->valid_utf_ystate.cnt = 10; + ystate = NULL; + } + else { + exec_context->valid_utf_ystate.yielded = 0; + restart_valid_utf: + ystate = &exec_context->valid_utf_ystate; + ystate->cnt = (int) extra_data->loop_limit; + } + errorcode = PRIV(yielding_valid_utf)((PCRE_PUCHAR)subject, length, + &erroroffset, ystate); +#endif if (errorcode != 0) { +#if defined(ERLANG_INTEGRATION) + if (ystate && ystate->yielded) { + ERTS_UPDATE_CONSUMED(extra_data, NULL); + SWAPOUT(); + return PCRE_ERROR_LOOP_LIMIT; + } +#endif if (offsetcount >= 2) { offsets[0] = erroroffset; @@ -6890,6 +6956,11 @@ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) return PCRE_ERROR_BADUTF8_OFFSET; #endif } +#if defined(ERLANG_INTEGRATION) +else { + exec_context->valid_utf_ystate.cnt = 0; +} +#endif #endif /* If the pattern was successfully studied with JIT support, run the JIT @@ -6950,7 +7021,11 @@ if (extra_data != NULL) #ifdef ERLANG_INTEGRATION if ((flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { - md->loop_limit = extra_data->loop_limit; + md->loop_limit = extra_data->loop_limit; + if (extra_data->restart_data) + md->loop_limit -= extra_data->loop_limit - exec_context->valid_utf_ystate.cnt; + if (md->loop_limit < 10) + md->loop_limit = 10; /* At least do something if we've come this far... */ } #endif } @@ -7266,14 +7341,8 @@ for(;;) #endif if ((start_bits[c/8] & (1 << (c&7))) != 0) { -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif - break; + ERTS_UPDATE_CONSUMED(extra_data, md); + break; } start_match++; } @@ -7298,13 +7367,7 @@ for(;;) (pcre_uint32)(end_subject - start_match) < study->minlength) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7353,13 +7416,7 @@ for(;;) if (p >= end_subject) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7390,11 +7447,7 @@ for(;;) EDEBUGF(("Calling match...")); rc = match(start_match, md->start_code, start_match, 2, md, NULL, 0); #ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } + ERTS_UPDATE_CONSUMED(extra_data, md); SWAPOUT(); while(rc == PCRE_ERROR_LOOP_LIMIT) { EDEBUGF(("Loop limit break detected")); diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h index c84dcb5a38..71f473e86f 100644 --- a/erts/emulator/pcre/pcre_internal.h +++ b/erts/emulator/pcre/pcre_internal.h @@ -2756,6 +2756,17 @@ extern int PRIV(strcmp_uc_c8_utf)(const pcre_uchar *, #endif /* COMPILE_PCRE[8|16|32] */ +#if defined(ERLANG_INTEGRATION) +struct PRIV(valid_utf_ystate) { + unsigned int cnt; + int length; + int yielded; + PCRE_PUCHAR p; +}; +extern int PRIV(yielding_valid_utf)(PCRE_PUCHAR, int, int *, + struct PRIV(valid_utf_ystate) *); +#endif + extern const pcre_uchar *PRIV(find_bracket)(const pcre_uchar *, BOOL, int); extern BOOL PRIV(is_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, int *, BOOL); diff --git a/erts/emulator/pcre/pcre_valid_utf8.c b/erts/emulator/pcre/pcre_valid_utf8.c index 516d8f4725..1dc1f9ba0c 100644 --- a/erts/emulator/pcre/pcre_valid_utf8.c +++ b/erts/emulator/pcre/pcre_valid_utf8.c @@ -107,19 +107,80 @@ Returns: = 0 if the string is a valid UTF-8 string int PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) { + +#if defined(ERLANG_INTEGRATION) + return PRIV(yielding_valid_utf)(string, length, erroroffset, NULL); +} + +int +PRIV(yielding_valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset, struct PRIV(valid_utf_ystate) *ystate) +{ +#endif + #ifdef SUPPORT_UTF register PCRE_PUCHAR p; +#if defined(ERLANG_INTEGRATION) +register long cnt; + +if (!ystate) { + cnt = -1; +} +else { + cnt = ystate->cnt; + if (ystate->yielded) { + p = ystate->p; + length = ystate->length; + if (length < 0) + goto restart_length; + else + goto restart_validate; + } +} +#endif + if (length < 0) { - for (p = string; *p != 0; p++); - length = (int)(p - string); + for (p = string; *p != 0; p++) { +#if defined(ERLANG_INTEGRATION) + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + restart_length: + (void) !0; +#endif + } + length = (int)(p - string); } for (p = string; length-- > 0; p++) { register pcre_uchar ab, c, d; +#if defined(ERLANG_INTEGRATION) + + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + + restart_validate: + +#endif + c = *p; if (c < 128) continue; /* ASCII character */ @@ -290,6 +351,14 @@ for (p = string; length-- > 0; p++) } } +#if defined(ERLANG_INTEGRATION) +if (ystate) { + /* Return with cnt set to amount consumed... */ + ystate->cnt -= cnt; + ystate->yielded = 0; +} +#endif + #else /* Not SUPPORT_UTF */ (void)(string); /* Keep picky compilers happy */ (void)(length); diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 7885d35d9d..9dcdd60060 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -1400,6 +1400,10 @@ get_conflicting_unicode_atoms(CIX, N) -> %% The message_latency_large tests that small distribution messages are %% not blocked by other large distribution messages. Basically it tests %% that fragmentation of distribution messages works. +%% +%% Because of large problems to get reliable values from these testcases +%% they no longer fail when the latency is incorrect. However, they are +%% kept as they continue to find bugs in the distribution implementation. message_latency_large_message(Config) when is_list(Config) -> measure_latency_large_message(?FUNCTION_NAME, fun(Dropper, Payload) -> Dropper ! Payload end). @@ -1484,7 +1488,11 @@ measure_latency_large_message(Nodename, DataFun) -> case {lists:max(Times), lists:min(Times)} of {Max, Min} when Max * 0.25 > Min, BuildType =:= opt -> - ct:fail({incorrect_latency, IndexTimes}); + %% We only issue a comment for this failure as the + %% testcases proved very difficult to run successfully + %% on many platforms. + ct:comment({incorrect_latency, IndexTimes}), + ok; _ -> ok end. @@ -1503,10 +1511,7 @@ measure_latency(DataFun, Dropper, Echo, Payload) -> end end) || _ <- lists:seq(1,2)], - [receive - {monitor, _Sender, busy_dist_port, _Info} -> - ok - end || _ <- lists:seq(1,10)], + wait_for_busy_dist(2 * 60 * 1000, 10), {TS, Times} = timer:tc(fun() -> @@ -1530,6 +1535,18 @@ measure_latency(DataFun, Dropper, Echo, Payload) -> end || {Sender, Ref} <- Senders], TS. +wait_for_busy_dist(_Tmo, 0) -> + ok; +wait_for_busy_dist(Tmo, N) -> + T0 = erlang:monotonic_time(millisecond), + receive + {monitor, _Sender, busy_dist_port, _Info} -> + wait_for_busy_dist(Tmo - (erlang:monotonic_time(millisecond) - T0), N - 1) + after Tmo -> + ct:log("Timed out waiting for busy_dist, ~p left",[N]), + timeout + end. + flush() -> receive _ -> @@ -2600,7 +2617,7 @@ verify_nc(Node) -> demonitor(MonRef,[flush]), ok; {Ref, Error} -> - ct:log("~p",[Error]), + ct:log("~s",[Error]), ct:fail(failed_nc_refc_check); {'DOWN', MonRef, _, _, _} = Down -> ct:log("~p",[Down]), diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl index 9f8ac42fa9..b7da69e556 100644 --- a/erts/emulator/test/dump_SUITE.erl +++ b/erts/emulator/test/dump_SUITE.erl @@ -140,13 +140,13 @@ free_dump(Config) when is_list(Config) -> {ok, NodeA} = start_node(Config), {ok, NodeB} = start_node(Config), - Self = self(), PidA = spawn_link( NodeA, fun() -> Self ! ready, + Reason = lists:duplicate(1000000,100), receive ok -> spawn(fun() -> @@ -154,24 +154,29 @@ free_dump(Config) when is_list(Config) -> timer:sleep(5), receive M -> - io:format("~p",[M]), - erlang:halt("dump") - end + io:format("~p",[M]) +%% We may want to add this timeout here in-case no busy condition is triggered +%% after 60 * 1000 -> +%% io:format("Timeout") + end, + erlang:halt("dump") end), - exit(lists:duplicate(1000000,100)) + exit(Reason) end end), - spawn_link(NodeB, - fun() -> - [erlang:monitor(process, PidA) || _ <- lists:seq(1,10000)], - Self ! done, - receive _ -> ok end - end), + PidB = spawn_link(NodeB, + fun() -> + [erlang:monitor(process, PidA) || _ <- lists:seq(1,10000)], + Self ! done, + receive _ -> ok end + end), receive done -> ok end, true = rpc:call(NodeA, os, putenv, ["ERL_CRASH_DUMP",Dump]), - ct:pal("~p",[rpc:call(NodeA, distribution_SUITE, make_busy, [NodeB, 1000])]), + %% Make the node busy towards NodeB for 10 seconds. + BusyPid = rpc:call(NodeA, distribution_SUITE, make_busy, [NodeB,10000]), + ct:pal("~p",[BusyPid]), receive ready -> unlink(PidA), PidA ! ok end, @@ -185,6 +190,10 @@ free_dump(Config) when is_list(Config) -> file:delete(Dump), + unlink(PidB), + + rpc:call(NodeB, erlang, halt, [0]), + ok. diff --git a/erts/emulator/test/multi_load_SUITE.erl b/erts/emulator/test/multi_load_SUITE.erl index edf3205812..c79e2b6dcd 100644 --- a/erts/emulator/test/multi_load_SUITE.erl +++ b/erts/emulator/test/multi_load_SUITE.erl @@ -30,7 +30,15 @@ all() -> [many,on_load,errors]. many(_Config) -> - Ms = make_modules(100, fun many_module/1), + + N = case erlang:system_info(build_type) of + valgrind -> + 10; + _ -> + 100 + end, + + Ms = make_modules(N, fun many_module/1), io:put_chars("Light load\n" "=========="), diff --git a/erts/emulator/test/socket_SUITE.erl b/erts/emulator/test/socket_SUITE.erl index d8cb1013e9..c82e2efad9 100644 --- a/erts/emulator/test/socket_SUITE.erl +++ b/erts/emulator/test/socket_SUITE.erl @@ -103,6 +103,7 @@ api_opt_simple_otp_options/1, api_opt_simple_otp_rcvbuf_option/1, api_opt_simple_otp_controlling_process/1, + api_opt_ip_add_drop_membership/1, %% *** API Operation Timeout *** api_to_connect_tcp4/1, @@ -585,6 +586,8 @@ groups() -> {api_basic, [], api_basic_cases()}, {api_async, [], api_async_cases()}, {api_options, [], api_options_cases()}, + {api_options_otp, [], api_options_otp_cases()}, + {api_options_ip, [], api_options_ip_cases()}, {api_op_with_timeout, [], api_op_with_timeout_cases()}, {socket_closure, [], socket_closure_cases()}, {sc_ctrl_proc_exit, [], sc_cp_exit_cases()}, @@ -700,11 +703,22 @@ api_async_cases() -> api_options_cases() -> [ + {group, api_options_otp}, + {group, api_options_ip} + ]. + +api_options_otp_cases() -> + [ api_opt_simple_otp_options, api_opt_simple_otp_rcvbuf_option, api_opt_simple_otp_controlling_process ]. +api_options_ip_cases() -> + [ + api_opt_ip_add_drop_membership + ]. + api_op_with_timeout_cases() -> [ api_to_connect_tcp4, @@ -7475,7 +7489,7 @@ api_opt_simple_otp_controlling_process(suite) -> api_opt_simple_otp_controlling_process(doc) -> []; api_opt_simple_otp_controlling_process(_Config) when is_list(_Config) -> - ?TT(?SECS(5)), + ?TT(?SECS(30)), tc_try(api_opt_simple_otp_controlling_process, fun() -> api_opt_simple_otp_controlling_process() end). @@ -7720,6 +7734,320 @@ api_opt_simple_otp_controlling_process() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% Tests that the add_mambership and drop_membership ip options work. +%% We create one server and two clients. The server only send messages, +%% the clients only receives messages. +%% An UDP datagram is forbidden (RFC 1122) from having a source address +%% that is a multicast address (or a broadcast address). +%% So, the server create a socket "for sending" and the clients sockets +%% "for receiving". +%% Sending socket: Bound to the local address (and any port). +%% When sending, the dest will be the multicast address +%% and port of the receiving socket. +%% Receiving socket: Bound to the multicast address and port. +api_opt_ip_add_drop_membership(suite) -> + []; +api_opt_ip_add_drop_membership(doc) -> + ["OTP-15908 (ERL-980)"]; +api_opt_ip_add_drop_membership(_Config) when is_list(_Config) -> + ?TT(?SECS(30)), + tc_try(api_opt_ip_add_drop_membership, + fun() -> + has_ip_add_membership_support(), + has_ip_drop_membership_support(), + has_ip_multicast_support() + end, + fun() -> api_opt_ip_add_drop_membership() end). + + +api_opt_ip_add_drop_membership() -> + Set = fun(S, Key, Val) -> + socket:setopt(S, ip, Key, Val) + end, + AddMembership = fun(S, Val) -> Set(S, add_membership, Val) end, + DropMembership = fun(S, Val) -> Set(S, drop_membership, Val) end, + + ServerSeq = + [ + %% *** Wait for start order part *** + #{desc => "await start", + cmd => fun(State) -> + {Tester, MSA} = ?SEV_AWAIT_START(), + {ok, State#{tester => Tester, msa => MSA}} + end}, + #{desc => "monitor tester", + cmd => fun(#{tester := Tester} = _State) -> + _MRef = erlang:monitor(process, Tester), + ok + end}, + + %% *** Init part *** + #{desc => "which local address", + cmd => fun(#{domain := Domain} = State) -> + LSA = which_local_socket_addr(Domain), + {ok, State#{local_sa => LSA}} + end}, + #{desc => "create socket", + cmd => fun(#{domain := Domain} = State) -> + case socket:open(Domain, dgram, udp) of + {ok, Sock} -> + {ok, State#{sock => Sock}}; + {error, _} = ERROR -> + ERROR + end + end}, + #{desc => "make recv socket reuse addr", + cmd => fun(#{sock := Sock} = _State) -> + case socket:setopt(Sock, socket, reuseaddr, true) of + ok -> + ok; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Failed set reuseaddr: " + "~n ~p", [Reason]), + ERROR + end + end}, + #{desc => "bind recv socket to multicast address", + cmd => fun(#{sock := Sock, msa := MSA} = State) -> + case socket:bind(Sock, MSA) of + {ok, Port} -> + ?SEV_IPRINT("bound to:" + "~n ~p", [Port]), + {ok, State#{msa => MSA#{port => Port}}}; + {error, _} = ERROR -> + ERROR + end + end}, + #{desc => "announce ready (init)", + cmd => fun(#{tester := Tester}) -> + ?SEV_ANNOUNCE_READY(Tester, init), + ok + end}, + + %% The actual test + #{desc => "await continue (add_membership)", + cmd => fun(#{tester := Tester} = _State) -> + ?SEV_AWAIT_CONTINUE(Tester, tester, add_membership) + end}, + #{desc => "add membership", + cmd => fun(#{sock := Sock, + msa := #{addr := MAddr}, + local_sa := #{addr := Addr}} = State) -> + MReq = #{multiaddr => MAddr, + interface => Addr}, + ?SEV_IPRINT("try add membership to:" + "~n ~p", [MReq]), + case AddMembership(Sock, MReq) of + ok -> + ?SEV_IPRINT("membership added"), + {ok, State#{mreq => MReq}}; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Failed adding membership to: " + "~n ~p" + "~n Reason: ~p", + [MReq, Reason]), + ERROR + end + end}, + #{desc => "announce ready (add-membership)", + cmd => fun(#{tester := Tester}) -> + ?SEV_ANNOUNCE_READY(Tester, add_membership), + ok + end}, + + #{desc => "await continue (drop_membership)", + cmd => fun(#{tester := Tester} = _State) -> + ?SEV_AWAIT_CONTINUE(Tester, tester, drop_membership) + end}, + #{desc => "drop membership", + cmd => fun(#{sock := Sock, + mreq := MReq} = State) -> + ?SEV_IPRINT("try drop membership from:" + "~n ~p", [MReq]), + case DropMembership(Sock, MReq) of + ok -> + ?SEV_IPRINT("membership dropped"), + {ok, maps:remove(mreq, State)}; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Failed drop membership from: " + "~n ~p" + "~n Reason: ~p", + [MReq, Reason]), + ERROR + end + end}, + #{desc => "announce ready (drop-membership)", + cmd => fun(#{tester := Tester}) -> + ?SEV_ANNOUNCE_READY(Tester, drop_membership), + ok + end}, + + + %% Termination + #{desc => "await terminate (from tester)", + cmd => fun(#{tester := Tester} = State) -> + case ?SEV_AWAIT_TERMINATE(Tester, tester) of + ok -> + {ok, maps:remove(tester, State)}; + {error, _} = ERROR -> + ERROR + end + end}, + #{desc => "close socket", + cmd => fun(#{sock := Sock} = State) -> + socket:close(Sock), + {ok, maps:remove(sock, State)} + end}, + + %% *** We are done *** + ?SEV_FINISH_NORMAL + ], + + + TesterSeq = + [ + %% *** Init part *** + #{desc => "monitor server", + cmd => fun(#{server := Server} = _State) -> + _MRef = erlang:monitor(process, Server), + ok + end}, + + %% Start the server + #{desc => "order server start", + cmd => fun(#{server := Pid, msa := MSA} = _State) -> + ?SEV_ANNOUNCE_START(Pid, MSA), + ok + end}, + #{desc => "await server ready (init)", + cmd => fun(#{server := Pid} = _State) -> + case ?SEV_AWAIT_READY(Pid, server, init) of + ok -> + ok; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Start of server failed: " + "~n ~p", [Reason]), + ERROR + end + end}, + + + %% *** The actual test *** + #{desc => "order server to continue (add-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_ANNOUNCE_CONTINUE(Server, add_membership), + ok + end}, + #{desc => "await server ready (add-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_AWAIT_READY(Server, server, add_membership) + end}, + + #{desc => "order server to continue (drop-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_ANNOUNCE_CONTINUE(Server, drop_membership), + ok + end}, + #{desc => "await server ready (drop-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_AWAIT_READY(Server, server, drop_membership) + end}, + + ?SEV_SLEEP(?SECS(1)), + + %% *** Termination *** + #{desc => "order server terminate", + cmd => fun(#{server := Server} = _State) -> + ?SEV_ANNOUNCE_TERMINATE(Server), + ok + end}, + #{desc => "await server termination", + cmd => fun(#{server := Server} = State) -> + ?SEV_AWAIT_TERMINATION(Server), + {ok, maps:remove(server, State)} + end}, + + %% *** We are done *** + ?SEV_FINISH_NORMAL + ], + + + i("get multicast address"), + Domain = inet, + MAddr = which_ip_multicast_address(), + MSA = #{family => Domain, addr => MAddr}, + + i("start server evaluator"), + ServerInitState = #{domain => Domain}, + Server = ?SEV_START("server", ServerSeq, ServerInitState), + + i("start tester evaluator"), + TesterInitState = #{domain => Domain, + msa => MSA, + server => Server#ev.pid}, + Tester = ?SEV_START("tester", TesterSeq, TesterInitState), + + i("await evaluator(s)"), + ok = ?SEV_AWAIT_FINISH([Tester, Server]). + + + +which_ip_multicast_address() -> + which_multicast_address(inet). + +which_multicast_address(Domain) -> + case os:type() of + {unix, linux} -> + WhichMAddr = fun([_, _, MAddr]) -> MAddr end, + which_multicast_address2(Domain, WhichMAddr); + + {unix, sunos} -> + WhichMAddr = fun([_, MAddr, _]) -> MAddr end, + which_multicast_address2(Domain, WhichMAddr); + + Type -> + not_supported({multicast, Type}) + end. + +%% Note that the 'netstat -g' table looks different on linux and SunOS +%% Linux: IfName - RefCnt - Group +%% SunOS: IfName - Group - RefCnt + +which_multicast_address2(Domain, WhichMAddr) -> + IfName = which_local_host_ifname(Domain), + NetstatGroupsStr = os:cmd("netstat -g | grep " ++ IfName), + NetstatGroups0 = string:tokens(NetstatGroupsStr, [$\n]), + NetstatGroups = [string:tokens(G, [$ ]) || G <- NetstatGroups0], + MAddrs = [WhichMAddr(NetstatGroup) || NetstatGroup <- + NetstatGroups], + which_multicast_address3(Domain, MAddrs). + +which_multicast_address3(_Domain, []) -> + not_supported({multicast, no_valid_addrs}); +which_multicast_address3(Domain, [MAddrStr|MAddrs]) -> + %% Even on linux some of these are not actually addresses, but + %% "host names", such as all-systems.mcast.net. But both + %% address strings, such as "224.0.0.251" and host name strings + %% gets translated into an address by the inet:inet:getaddr/2. + case inet:getaddr(MAddrStr, Domain) of + {ok, MAddr} -> + MAddr; + {error, _} -> + which_multicast_address3(Domain, MAddrs) + end. + +which_local_host_ifname(Domain) -> + case which_local_host_info(Domain) of + {ok, {Name, _Addr, _Flags}} -> + Name; + {error, Reason} -> + not_supported({multicast, Reason}) + end. + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% API OPERATIONS WITH TIMEOUT %% @@ -16480,7 +16808,7 @@ traffic_ping_pong_send_and_receive_udp2(InitState) -> LSA = which_local_socket_addr(Domain), {ok, State#{local_sa => LSA}} end}, - #{desc => "create listen socket", + #{desc => "create socket", cmd => fun(#{domain := Domain, proto := Proto} = State) -> case socket:open(Domain, dgram, Proto) of {ok, Sock} -> @@ -24032,70 +24360,115 @@ which_local_socket_addr(local = Domain) -> %% We should really implement this using the (new) net module, %% but until that gets the necessary functionality... which_local_socket_addr(Domain) -> - case inet:getifaddrs() of - {ok, IFL} -> - Addr = which_addr(Domain, IFL), + case which_local_host_info(Domain) of + {ok, {_Name, _Flags, Addr}} -> #{family => Domain, addr => Addr}; {error, Reason} -> - ?FAIL({inet, getifaddrs, Reason}) + ?FAIL(Reason) + end. + + +%% Returns the interface (name), flags and address (not 127...) +%% of the local host. +which_local_host_info(Domain) -> + case inet:getifaddrs() of + {ok, IFL} -> + which_local_host_info(Domain, IFL); + {error, _} = ERROR -> + ERROR end. -which_addr(_Domain, []) -> +which_local_host_info(_Domain, []) -> ?FAIL(no_address); -which_addr(Domain, [{"lo" ++ _, _}|IFL]) -> - which_addr(Domain, IFL); -which_addr(Domain, [{_Name, IFO}|IFL]) -> - case which_addr2(Domain, IFO) of - {ok, Addr} -> - Addr; - {error, no_address} -> - which_addr(Domain, IFL) +which_local_host_info(Domain, [{"lo" ++ _, _}|IFL]) -> + which_local_host_info(Domain, IFL); +which_local_host_info(Domain, [{"docker" ++ _, _}|IFL]) -> + which_local_host_info(Domain, IFL); +which_local_host_info(Domain, [{"br-" ++ _, _}|IFL]) -> + which_local_host_info(Domain, IFL); +which_local_host_info(Domain, [{Name, IFO}|IFL]) -> + case which_local_host_info2(Domain, IFO) of + {ok, {Flags, Addr}} -> + {ok, {Name, Flags, Addr}}; + {error, _} -> + which_local_host_info(Domain, IFL) end; -which_addr(Domain, [_|IFL]) -> - which_addr(Domain, IFL). +which_local_host_info(Domain, [_|IFL]) -> + which_local_host_info(Domain, IFL). -which_addr2(_Domain, []) -> +which_local_host_info2(Domain, IFO) -> + case lists:keysearch(flags, 1, IFO) of + {value, {flags, Flags}} -> + which_local_host_info2(Domain, IFO, Flags); + false -> + {error, no_flags} + end. + +which_local_host_info2(_Domain, [], _Flags) -> {error, no_address}; -which_addr2(inet = _Domain, [{addr, Addr}|_IFO]) +which_local_host_info2(inet = _Domain, [{addr, Addr}|_IFO], Flags) when (size(Addr) =:= 4) andalso (element(1, Addr) =/= 127) -> - {ok, Addr}; -which_addr2(inet6 = _Domain, [{addr, Addr}|_IFO]) + {ok, {Flags, Addr}}; +which_local_host_info2(inet6 = _Domain, [{addr, Addr}|_IFO], Flags) when (size(Addr) =:= 8) andalso (element(1, Addr) =/= 0) andalso (element(1, Addr) =/= 16#fe80) -> - {ok, Addr}; -which_addr2(Domain, [_|IFO]) -> - which_addr2(Domain, IFO). - + {ok, {Flags, Addr}}; +which_local_host_info2(Domain, [_|IFO], Flags) -> + which_local_host_info2(Domain, IFO, Flags). -unlink_path(Path) -> - unlink_path(Path, fun() -> ok end, fun() -> ok end). - -unlink_path(Path, Success, Failure) when is_list(Path) andalso - is_function(Success, 0) andalso - is_function(Failure, 0) -> - ?SEV_IPRINT("try unlink path: " - "~n ~s", [Path]), - case os:cmd("unlink " ++ Path) of - "" -> - ?SEV_IPRINT("path unlinked: " - "~n Path: ~s", [Path]), - Success(); - Result -> - ?SEV_EPRINT("unlink maybe failed: " - "~n Path: ~s" - "~n Res: ~s", [Path, Result]), - Failure() - end; -unlink_path(_, _, _) -> - ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Here are all the *general* test vase condition functions. +%% Here are all the *general* test case condition functions. + +%% We also need (be able) to figure out the the multicast address, +%% which we only support for some platforms (linux and sunos). +%% We don't do that here, but since we can only do that (find a +%% multicast address) for specific platforms, we check that we are +%% on of those platforms here. +has_ip_multicast_support() -> + case os:type() of + {unix, OsName} when (OsName =:= linux) orelse + (OsName =:= sunos) -> + case which_local_host_info(inet) of + {ok, {_Name, Flags, _Addr}} -> + case lists:member(multicast, Flags) of + true -> + ok; + false -> + not_supported(multicast) + end; + {error, Reason} -> + not_supported({multicast, Reason}) + end; + Type -> + not_supported({multicast, Type}) + end. + +has_ip_add_membership_support() -> + has_socket_option_ip_support(add_membership). + +has_ip_drop_membership_support() -> + has_socket_option_ip_support(drop_membership). + + +has_socket_option_ip_support(Opt) -> + has_socket_option_support(ip, Opt). + +has_socket_option_support(Level, Option) -> + case socket:supports(options, Level, Option) of + true -> + ok; + false -> + not_supported({options, Level, Option}) + end. + + + unix_domain_socket_host_cond() -> unix_domain_socket_host_cond(os:type(), os:version()). @@ -24135,6 +24508,34 @@ has_support_ipv6() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +unlink_path(Path) -> + unlink_path(Path, fun() -> ok end, fun() -> ok end). + +unlink_path(Path, Success, Failure) when is_list(Path) andalso + is_function(Success, 0) andalso + is_function(Failure, 0) -> + ?SEV_IPRINT("try unlink path: " + "~n ~s", [Path]), + case os:cmd("unlink " ++ Path) of + "" -> + ?SEV_IPRINT("path unlinked: " + "~n Path: ~s", [Path]), + Success(); + Result -> + ?SEV_EPRINT("unlink maybe failed: " + "~n Path: ~s" + "~n Res: ~s", [Path, Result]), + Failure() + end; +unlink_path(_, _, _) -> + ok. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +not_supported(What) -> + skip({not_supported, What}). + not_yet_implemented() -> skip("not yet implemented"). diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index c793243c13..477a501876 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -1690,9 +1690,9 @@ static char **build_args_from_string(char *string) for(;;) { switch (state) { case Start: - if (!*p) + if (!*p) goto done; - if (argc >= alloced - 1) { /* Make room for extra NULL */ + if (argc >= alloced - 2) { /* Make room for extra NULL and "--" */ argv = erealloc(argv, (alloced += 10) * sizeof(char *)); } cur_s = argc + argv; @@ -1781,11 +1781,14 @@ static char **build_args_from_string(char *string) } } done: - argv[argc] = NULL; /* Sure to be large enough */ if (!argc) { efree(argv); return NULL; } + argv[argc++] = "--"; /* Add a -- separator in order + for different from different environments + to effect each other */ + argv[argc++] = NULL; /* Sure to be large enough */ return argv; #undef ENSURE } @@ -2036,11 +2039,13 @@ initial_argv_massage(int *argc, char ***argv) argv_buf ab = {0}, xab = {0}; int ix, vix, ac; char **av; + char *sep = "--"; struct { int argc; char **argv; } avv[] = {{INT_MAX, NULL}, {INT_MAX, NULL}, {INT_MAX, NULL}, - {INT_MAX, NULL}, {INT_MAX, NULL}, {INT_MAX, NULL}}; + {INT_MAX, NULL}, {INT_MAX, NULL}, + {INT_MAX, NULL}, {INT_MAX, NULL}}; /* * The environment flag containing OTP release is intentionally * undocumented and intended for OTP internal use only. @@ -2060,6 +2065,8 @@ initial_argv_massage(int *argc, char ***argv) if (*argc > 1) { avv[vix].argc = *argc - 1; avv[vix++].argv = &(*argv)[1]; + avv[vix].argc = 1; + avv[vix++].argv = &sep; } av = build_args_from_env("ERL_FLAGS"); diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 710a7a9ef6..59de9bdec8 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -312,8 +312,11 @@ if [ "x$GDB" = "x" ]; then # on multiple cores (especially with async threads). Valgrind only run one pthread # at a time anyway so there is no point letting it utilize more than one core. # Use $sched_arg to force all schedulers online to emulate multicore. - taskset1="taskset 1" ncpu=`cat /proc/cpuinfo | grep -w processor | wc -l` + # Choose a random core in order to not collide with any other valgrind + # run on the same machine. + taskset1=$((1 << (`shuf -i 1-$ncpu -n 1` - 1) )) + taskset1="taskset $taskset1" sched_arg="-S$ncpu:$ncpu" else taskset1= diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 730f0a0c64..8b6abb5336 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -37,7 +37,7 @@ document etp-help % - GDB command toolbox for analyzing core dumps from the % Erlang emulator (BEAM). % -% Should work for 32-bit erts-5.2/R9B, ... +% Should work for 32-bit and 64-bit unix gdb % % The commands are prefixed with: % etp: Acronym for erts-term-print @@ -55,7 +55,8 @@ document etp-help % Special commands for not really terms: % etp-mfa, etp-cp, etp-disasm, % etp-msgq, etpf-msgq, -% etp-stacktrace, etp-stackdump, etpf-stackdump, etp-dictdump +% etp-stacktrace, etp-stacktrace-emu, etp-stackdump, etp-stackdump-emu, +% etpf-stackdump, etp-dictdump % etp-process-info, etp-process-memory-info % etp-port-info, etp-port-state, etp-port-sched-flags % etp-heapdump, etp-offheapdump, etpf-offheapdump, @@ -1452,30 +1453,74 @@ end define etp-stack-preamble set $etp_stack_p = ($arg0)->stop set $etp_stack_end = ($arg0)->hend + if ($arg0)->state.counter & 0x8000 + printf "%%%%%% WARNING: The process is currently running, so c_p->stop will not be correct\r\n" + printf "%%%%%% Consider using %s-emu instead\r\n", $arg1 + end printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p if ($arg0)->i != 0 - etp-1 ((Eterm)($arg0)->i) 0 - printf " (I)\n" + printf "I: " + etp ((Eterm)($arg0)->i) + end + if ($arg0)->cp != 0 + printf "cp:" + etp ((Eterm)($arg0)->cp) end +end + +define etp-stack-preamble-emu + set $etp_stack_p = E + set $etp_stack_end = ($arg0)->hend + printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p + printf "I: " + etp ((BeamInstr)I) if ($arg0)->cp != 0 - etp-1 ((Eterm)($arg0)->cp) 0 - printf " (cp)\n" + printf "cp: " + etp ((Eterm)($arg0)->cp) end end +define etp-stacktrace-1 + set $etp_stack_stop = (Eterm*)($arg0) + set $etp_stack_send = (Eterm*)($arg1) + set $etp_stack_cnt = 0 + while $etp_stack_stop < $etp_stack_send + if ($etp_stack_stop[0] & 0x3) == 0x0 + # Continuation pointer + printf "%d: ", $etp_stack_cnt + etp $etp_stack_stop[0] + end + set $etp_stack_stop++ + set $etp_stack_cnt++ + end +end + +define etp-stacktrace-emu +# Args: Process* +# +# Non-reentrant +# + etp-stack-preamble-emu ($arg0) + etp-stacktrace-1 $etp_stack_p $etp_stack_end +end + +document etp-stacktrace-emu +%--------------------------------------------------------------------------- +% etp-stacktrace-emu Process* +% +% Take an Process* and print a stactrace for the process. +% This macro assumes that the current frame is the process_main frame +% and that E is not optimized out. +%--------------------------------------------------------------------------- +end + define etp-stacktrace # Args: Process* # # Non-reentrant # - etp-stack-preamble ($arg0) - while $etp_stack_p < $etp_stack_end - if ($etp_stack_p[0] & 0x3) == 0x0 - # Continuation pointer - etp $etp_stack_p[0] - end - set $etp_stack_p++ - end + etp-stack-preamble ($arg0) "etp-stacktrace" + etp-stacktrace-1 $etp_stack_p $etp_stack_end end document etp-stacktrace @@ -1488,16 +1533,48 @@ document etp-stacktrace %--------------------------------------------------------------------------- end +define etp-stackdump-1 + # Args: Eterm *stop, Eterm *hend + # + # Non-reentrant + # + set $etp_stackdump_stop = (Eterm*)($arg0) + set $etp_stackdump_send = (Eterm*)($arg1) + set $etp_stackdump_cnt = 0 + while $etp_stackdump_stop < $etp_stackdump_send + printf "%d: ", $etp_stackdump_cnt + etp $etp_stackdump_stop[0] + set $etp_stackdump_stop++ + set $etp_stackdump_cnt++ + end +end + +define etp-stackdump-emu +# Args: Process* +# +# Non-reentrant +# + etp-stack-preamble-emu ($arg0) + etp-stackdump-1 $etp_stack_p $etp_stack_end +end + +document etp-stacktrace-emu +%--------------------------------------------------------------------------- +% etp-stacktrace-emu Process* +% +% Take an Process* and print a stactdump for the process. +% This macro assumes that the current frame is the process_main frame +% and that E is not optimized out. +%--------------------------------------------------------------------------- +end + define etp-stackdump # Args: Process* # # Non-reentrant # - etp-stack-preamble ($arg0) - while $etp_stack_p < $etp_stack_end - etp $etp_stack_p[0] - set $etp_stack_p++ - end + etp-stack-preamble ($arg0) "etp-stackdump" + etp-stackdump-1 $etp_stack_p $etp_stack_end end document etp-stackdump @@ -2959,12 +3036,21 @@ define etp-fds end end +document etp-fds +%--------------------------------------------------------------------------- +% etp-fds +% +% Print the state of the fds currently in check_io. Only works in running systems. +%--------------------------------------------------------------------------- +end + define etp-disasm-1 - set $code_ptr = ((BeamInstr*)$arg0) - set $addr = *$code_ptr + set $code_ptr = ((BeamInstr*)($arg0)) + set $addr32 = (BeamInstr)(Uint32)*$code_ptr + set $addr64 = (BeamInstr)(Uint64)*$code_ptr set $i = 0 - while $i < (sizeof(opc) / sizeof(OpEntry)) - if $addr == beam_ops[$i] + while $i < num_instructions + if $addr32 == beam_ops[$i] || $addr64 == beam_ops[$i] printf "%s %d", opc[$i].name, opc[$i].sz set $next_i = $code_ptr + opc[$i].sz set $i += 4999 @@ -2974,6 +3060,11 @@ define etp-disasm-1 end define etp-disasm + if $argc == 1 + set $code_end = $arg0 + else + set $code_end = $arg1 + end etp-cp-func-info-1 $arg0 if $etp_cp_p == 0 printf "invalid argument" @@ -2982,7 +3073,7 @@ define etp-disasm printf ": " etp-disasm-1 $arg0 printf "\r\n" - while $next_i < ((BeamInstr*)$arg1) + while $next_i < ((BeamInstr*)$code_end) set $prev_i = $next_i etp-cp-func-info-1 $next_i etp-mfa-1 $etp_cp_p $cp_cp_p_offset @@ -2998,6 +3089,16 @@ define etp-disasm end end +document etp-disasm +%--------------------------------------------------------------------------- +% etp-fds BeamInstr* (BeamInstr*) +% +% Disassemble the instructions inbetween arg0 and arg1, +% if no second argument is given only the current +% instruction is printed. +%--------------------------------------------------------------------------- +end + ############################################################################ # # Timer Wheel diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl index 602dc5ce2e..952e6da4dc 100644 --- a/erts/test/erlexec_SUITE.erl +++ b/erts/test/erlexec_SUITE.erl @@ -30,7 +30,7 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). -export([args_file/1, evil_args_file/1, env/1, args_file_env/1, - otp_7461/1, otp_7461_remote/1, otp_8209/1, + otp_7461/1, otp_7461_remote/1, argument_separation/1, zdbbl_dist_buf_busy_limit/1]). -include_lib("common_test/include/ct.hrl"). @@ -51,21 +51,28 @@ suite() -> all() -> [args_file, evil_args_file, env, args_file_env, - otp_7461, otp_8209, zdbbl_dist_buf_busy_limit]. + otp_7461, argument_separation, zdbbl_dist_buf_busy_limit]. %% Test that plain first argument does not -%% destroy -home switch [OTP-8209] -otp_8209(Config) when is_list(Config) -> +%% destroy -home switch [OTP-8209] or interact with environments +argument_separation(Config) when is_list(Config) -> {ok,[[PName]]} = init:get_argument(progname), SNameS = "erlexec_test_01", SName = list_to_atom(SNameS++"@"++ hd(tl(string:lexemes(atom_to_list(node()),"@")))), - Cmd = PName ++ " dummy_param -sname "++SNameS++" -setcookie "++ - atom_to_list(erlang:get_cookie()), - open_port({spawn,Cmd},[]), + Cmd = PName ++ " cmd_param -sname "++SNameS++" -setcookie "++ + atom_to_list(erlang:get_cookie()) ++ " -cmd_test", + open_port({spawn,Cmd},[{env,[{"ERL_AFLAGS","-atest"}, + {"ERL_FLAGS","env_param -test"}, + {"ERL_ZFLAGS","zenv_param"}]}]), pong = loop_ping(SName,40), + ct:log("emu_args: ~p",[rpc:call(SName,erlang,system_info,[emu_args])]), {ok,[[_]]} = rpc:call(SName,init,get_argument,[home]), - ["dummy_param"] = rpc:call(SName,init,get_plain_arguments,[]), + {ok,[[]]} = rpc:call(SName,init,get_argument,[atest]), + {ok,[[]]} = rpc:call(SName,init,get_argument,[cmd_test]), + {ok,[[]]} = rpc:call(SName,init,get_argument,[test]), + error = rpc:call(SName,init,get_argument,[unkown]), + ["cmd_param","env_param","zenv_param"] = rpc:call(SName,init,get_plain_arguments,[]), ok = cleanup_nodes(), ok. @@ -85,6 +92,7 @@ cleanup_node(SNameS,N) -> end. loop_ping(_,0) -> + flush(), pang; loop_ping(Node,N) -> case net_adm:ping(Node) of @@ -98,6 +106,14 @@ loop_ping(Node,N) -> pong end. +flush() -> + receive M -> + ct:pal("~p",[M]), + flush() + after 10 -> + ok + end. + args_file(Config) when is_list(Config) -> AFN1 = privfile("1", Config), AFN2 = privfile("2", Config), diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl index a9977b0b1d..6492d1e1bf 100644 --- a/lib/compiler/src/beam_ssa.erl +++ b/lib/compiler/src/beam_ssa.erl @@ -96,7 +96,8 @@ %% To avoid the collapsing, change the value of SET_LIMIT to 50 in the %% file erl_types.erl in the hipe application. --type prim_op() :: 'bs_add' | 'bs_extract' | 'bs_init' | 'bs_init_writable' | +-type prim_op() :: 'bs_add' | 'bs_extract' | 'bs_get_tail' | + 'bs_init' | 'bs_init_writable' | 'bs_match' | 'bs_put' | 'bs_start_match' | 'bs_test_tail' | 'bs_utf16_size' | 'bs_utf8_size' | 'build_stacktrace' | 'call' | 'catch_end' | @@ -117,11 +118,12 @@ '+' | '-' | '*' | '/'. %% Primops only used internally during code generation. --type cg_prim_op() :: 'bs_get' | 'bs_match_string' | 'bs_restore' | 'bs_skip' | +-type cg_prim_op() :: 'bs_get' | 'bs_get_position' | 'bs_match_string' | + 'bs_restore' | 'bs_save' | 'bs_set_position' | 'bs_skip' | 'copy' | 'put_tuple_arity' | 'put_tuple_element' | - 'set_tuple_element'. + 'put_tuple_elements' | 'set_tuple_element'. --import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1]). +-import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1,umerge/1]). -spec add_anno(Key, Value, Construct) -> Construct when Key :: atom(), @@ -649,12 +651,18 @@ is_commutative('=/=') -> true; is_commutative('/=') -> true; is_commutative(_) -> false. -def_used_1([#b_blk{is=Is,last=Last}|Bs], Preds, Def0, Used0) -> - {Def,Used1} = def_used_is(Is, Preds, Def0, Used0), - Used = ordsets:union(used(Last), Used1), - def_used_1(Bs, Preds, Def, Used); -def_used_1([], _Preds, Def, Used) -> - {ordsets:from_list(Def),Used}. +def_used_1([#b_blk{is=Is,last=Last}|Bs], Preds, Def0, UsedAcc) -> + {Def,Used} = def_used_is(Is, Preds, Def0, used(Last)), + case Used of + [] -> + def_used_1(Bs, Preds, Def, UsedAcc); + [_|_] -> + def_used_1(Bs, Preds, Def, [Used|UsedAcc]) + end; +def_used_1([], _Preds, Def0, UsedAcc) -> + Def = ordsets:from_list(Def0), + Used = umerge(UsedAcc), + {Def,Used}. def_used_is([#b_set{op=phi,dst=Dst,args=Args}|Is], Preds, Def0, Used0) -> diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index 62cd5b5120..bc28f58712 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -263,7 +263,7 @@ %% @see subtrees/1 %% @see meta/1 --type ctype() :: 'alias' | 'apply' | 'binary' | 'bitrst' | 'call' | 'case' +-type ctype() :: 'alias' | 'apply' | 'binary' | 'bitstr' | 'call' | 'case' | 'catch' | 'clause' | 'cons' | 'fun' | 'let' | 'letrec' | 'literal' | 'map' | 'map_pair' | 'module' | 'primop' | 'receive' | 'seq' | 'try' | 'tuple' | 'values' | 'var'. diff --git a/lib/crypto/c_src/Makefile.in b/lib/crypto/c_src/Makefile.in index 2512013ed6..f922c3fb9b 100644 --- a/lib/crypto/c_src/Makefile.in +++ b/lib/crypto/c_src/Makefile.in @@ -100,8 +100,7 @@ CRYPTO_OBJS = $(OBJDIR)/crypto$(TYPEMARKER).o \ $(OBJDIR)/srp$(TYPEMARKER).o CALLBACK_OBJS = $(OBJDIR)/crypto_callback$(TYPEMARKER).o NIF_MAKEFILE = $(PRIVDIR)/Makefile -CRYPTO_STATIC_OBJS = $(OBJDIR)/crypto_static$(TYPEMARKER).o\ - $(OBJDIR)/crypto_callback_static$(TYPEMARKER).o +CRYPTO_STATIC_OBJS = $(patsubst $(OBJDIR)/%$(TYPEMARKER).o,$(OBJDIR)/%_static$(TYPEMARKER).o,$(CRYPTO_OBJS) $(CALLBACK_OBJS)) NIF_ARCHIVE = $(LIBDIR)/crypto$(TYPEMARKER).a diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c index a109f444cf..941e03cc98 100644 --- a/lib/crypto/c_src/api_ng.c +++ b/lib/crypto/c_src/api_ng.c @@ -334,12 +334,11 @@ ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg if ((ctx_res = enif_alloc_resource(evp_cipher_ctx_rtype, sizeof(struct evp_cipher_ctx))) == NULL) return EXCP_ERROR(env, "Can't allocate resource"); - if (!get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], + if (get_init_args(env, ctx_res, argv[0], argv[1], argv[2], argv[argc-1], &cipherp, &ret)) - /* Error msg in &ret */ - goto ret; + ret = enif_make_resource(env, ctx_res); + /* else error msg in ret */ - ret = enif_make_resource(env, ctx_res); if(ctx_res) enif_release_resource(ctx_res); } else if (enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) { diff --git a/lib/crypto/c_src/bn.c b/lib/crypto/c_src/bn.c index 34ed4f7ebc..6021d56db6 100644 --- a/lib/crypto/c_src/bn.c +++ b/lib/crypto/c_src/bn.c @@ -32,8 +32,6 @@ int get_bn_from_mpint(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) if (bin.size > INT_MAX - 4) goto err; - ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); - if (bin.size < 4) goto err; sz = (int)bin.size - 4; @@ -60,8 +58,6 @@ int get_bn_from_bin(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp) if (bin.size > INT_MAX) goto err; - ERL_VALGRIND_ASSERT_MEM_DEFINED(bin.data, bin.size); - if ((ret = BN_bin2bn(bin.data, (int)bin.size, NULL)) == NULL) goto err; @@ -103,8 +99,6 @@ ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) unsigned extra_byte; ERL_NIF_TERM ret; - ASSERT(argc == 4); - if (!get_bn_from_bin(env, argv[0], &bn_base)) goto bad_arg; if (!get_bn_from_bin(env, argv[1], &bn_exponent)) @@ -177,7 +171,6 @@ ERL_NIF_TERM bn2term(ErlNifEnv* env, const BIGNUM *bn) BN_bn2bin(bn, ptr); - ERL_VALGRIND_MAKE_MEM_DEFINED(ptr, dlen); return ret; err: diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index d4fe064edd..c1bc5ff597 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -605,11 +605,9 @@ ordinal(N) when is_integer(N) -> io_lib:format("~wth", [N]). %% Functions that parse type strings, literal strings, and contract %% strings. Return strings formatted by erl_pp. -%% If lib/hipe/cerl/erl_types.erl is compiled with DEBUG=true, -%% the contents of opaque types are showed inside brackets. -%% Since erl_parse:parse_form() cannot handle the bracket syntax, -%% no indentation is done. -%%-define(DEBUG , true). +%% Note we always have to catch any error when trying to parse +%% the syntax because other BEAM languages may not emit an +%% Erlang AST that transforms into valid Erlang Source Code. -define(IND, 10). @@ -620,13 +618,15 @@ con(M, F, Src, I) -> sig(Src, false) -> Src; sig(Src, true) -> - Str = lists:flatten(io_lib:format("-spec ~w:~tw~ts.", [a, b, Src])), - {ok, Tokens, _EndLocation} = erl_scan:string(Str), - exec(fun() -> - {ok, {attribute, _, spec, {_MFA, Types}}} = - erl_parse:parse_form(Tokens), - indentation(?IND) ++ pp_spec(Types) - end, Src). + try + Str = lists:flatten(io_lib:format("-spec ~w:~tw~ts.", [a, b, Src])), + {ok, Tokens, _EndLocation} = erl_scan:string(Str), + {ok, {attribute, _, spec, {_MFA, Types}}} = + erl_parse:parse_form(Tokens), + indentation(?IND) ++ pp_spec(Types) + catch + _:_ -> Src + end. %% Argument(list)s are a mix of types and Erlang code. Note: sometimes %% (contract_range, call_without_opaque, opaque_type_test), the initial @@ -681,21 +681,15 @@ ts(Src) -> [C1|Src1] = Src, % $< (product) or $( (arglist) [C2|RevSrc2] = lists:reverse(Src1), Src2 = lists:reverse(RevSrc2), - exec(fun() -> - Types = parse_types_and_literals(Src2), - CommaInd = [$, | Ind], - (indentation(?IND-1) ++ - [C1 | lists:join(CommaInd, [pp_type(Type) || Type <- Types])] ++ - [C2]) - end, Src). - --ifdef(DEBUG). -exec(F, R) -> - try F() catch _:_ -> R end. --else. -exec(F, _) -> - F(). --endif. + try + Types = parse_types_and_literals(Src2), + CommaInd = [$, | Ind], + (indentation(?IND-1) ++ + [C1 | lists:join(CommaInd, [pp_type(Type) || Type <- Types])] ++ + [C2]) + catch + _:_ -> Src + end. indentation(I) -> [$\n | lists:duplicate(I, $\s)]. diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 24a205ced9..9967488f61 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -317,7 +317,7 @@ store_cookies(SetCookieHeaders, Url, Profile) {error, Bad, _} -> {error, {parse_failed, Bad}}; URI -> - Scheme = scheme_to_atom(maps:get(scheme, URI, '')), + Scheme = scheme_to_atom(maps:get(scheme, URI, undefined)), Host = maps:get(host, URI, ""), Port = maps:get(port, URI, default_port(Scheme)), Path = uri_string:recompose(#{path => maps:get(path, URI, "")}), @@ -536,7 +536,7 @@ handle_request(Method, Url, BracketedHost = proplists:get_value(ipv6_host_with_brackets, Options), - Scheme = scheme_to_atom(maps:get(scheme, URI, '')), + Scheme = scheme_to_atom(maps:get(scheme, URI, undefined)), Userinfo = maps:get(userinfo, URI, ""), Host = http_util:maybe_add_brackets(maps:get(host, URI, ""), BracketedHost), Port = maps:get(port, URI, default_port(Scheme)), @@ -591,8 +591,8 @@ scheme_to_atom("http") -> http; scheme_to_atom("https") -> https; -scheme_to_atom('') -> - ''; +scheme_to_atom(undefined) -> + throw({error, {no_scheme}}); scheme_to_atom(Scheme) -> throw({error, {bad_scheme, Scheme}}). diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index d4b33ae2c6..1d37e71847 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -106,7 +106,8 @@ real_requests()-> streaming_error, inet_opts, invalid_headers, - invalid_body + invalid_body, + no_scheme ]. real_requests_esi() -> @@ -1231,6 +1232,16 @@ invalid_body(Config) -> ok end. + +%%------------------------------------------------------------------------- + +no_scheme(_Config) -> + {error,{bad_scheme,"ftp"}} = httpc:request("ftp://foobar"), + {error,{no_scheme}} = httpc:request("//foobar"), + {error,{no_scheme}} = httpc:request("foobar"), + ok. + + %%------------------------------------------------------------------------- remote_socket_close(Config) when is_list(Config) -> URL = url(group_name(Config), "/just_close.html", Config), diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index 1aa4b7a3a2..5aa2caadf0 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -89,8 +89,8 @@ <p>Filter functions can be used for more sophisticated filtering than the log level check provides. A filter function can stop or pass a log event, based on any of the event's contents. It can - also modify all parts of the log event. See see - section <seealso marker="#filters">Filters</seealso> for more + also modify all parts of the log event. See section + <seealso marker="#filters">Filters</seealso> for more details.</p> <p>If a log event passes through all primary filters and all handler filters for a specific handler, Logger forwards the diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index 7940903658..24aff83fbd 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -1645,6 +1645,7 @@ fmt_addr({ok,Addr}, Proto) -> {{0,0,0,0,0,0,0,0},Port} -> "*:" ++ fmt_port(Port, Proto); {{127,0,0,1},Port} -> "localhost:" ++ fmt_port(Port, Proto); {{0,0,0,0,0,0,0,1},Port} -> "localhost:" ++ fmt_port(Port, Proto); + {local, Path} -> "local:" ++ binary_to_list(Path); {IP,Port} -> inet_parse:ntoa(IP) ++ ":" ++ fmt_port(Port, Proto) end. diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 3fd6eae423..8b7cb4dcd4 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -347,6 +347,7 @@ <datatype> <name name="subsystem_daemon_option"/> + <name name="subsystem_specs"/> <name name="subsystem_spec"/> <desc> <p>Defines a subsystem in the daemon.</p> diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 54e98ee10e..a991f72cf2 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -312,7 +312,8 @@ | gen_tcp:listen_option() | ?COMMON_OPTION . --type subsystem_daemon_option() :: {subsystems, subsystem_spec()}. +-type subsystem_daemon_option() :: {subsystems, subsystem_specs()}. +-type subsystem_specs() :: [ subsystem_spec() ]. -type shell_daemon_option() :: {shell, mod_fun_args() | 'shell_fun/1'() | 'shell_fun/2'() }. -type 'shell_fun/1'() :: fun((User::string()) -> pid()) . @@ -478,14 +479,6 @@ recv_ext_info }). --record(ssh_key, - { - type, - public, - private, - comment = "" - }). - -record(ssh_pty, {term = "", % e.g. "xterm" width = 80, height = 25, @@ -493,13 +486,6 @@ pixel_height = 768, modes = <<>>}). -%% assertion macro --define(ssh_assert(Expr, Reason), - case Expr of - true -> ok; - _ -> exit(Reason) - end). - %% dbg help macros -define(wr_record(N,BlackList), diff --git a/lib/ssl/doc/src/standards_compliance.xml b/lib/ssl/doc/src/standards_compliance.xml index 3a472d4776..9df48b99d3 100644 --- a/lib/ssl/doc/src/standards_compliance.xml +++ b/lib/ssl/doc/src/standards_compliance.xml @@ -135,8 +135,10 @@ <item>Groups: all standard groups supported for the Diffie-Hellman key exchange</item> <item>Ciphers: TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256 and TLS_AES_128_CCM_SHA256</item> - <item>Signature Algorithms: RSA and RSA PSS</item> - <item>Certificates: currently only certificates with RSA keys are supported</item> + <item>Signature Algorithms: rsa_pkcs1_sha256, rsa_pkcs1_sha384, rsa_pkcs1_sha512, + ecdsa_secp256r1_sha256, ecdsa_secp384r1_sha384, ecdsa_secp521r1_sha512, rsa_pss_rsae_sha256, + rsa_pss_rsae_sha384, rsa_pss_rsae_sha512, rsa_pkcs1_sha1 and ecdsa_sha1</item> + <item>Certificates: RSA (it MUST use the rsaEncryption OID) and ECDSA keys</item> </list> <p>Other notable features:</p> <list type="bulleted"> @@ -727,20 +729,20 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -830,20 +832,20 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -1956,8 +1958,8 @@ </url> </cell> <cell align="left" valign="middle"><em></em></cell> - <cell align="left" valign="middle"><em>PC</em></cell> - <cell align="left" valign="middle"><em>22</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -1981,8 +1983,8 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle"><em>Digital signatures</em></cell> - <cell align="left" valign="middle"><em>PC</em></cell> - <cell align="left" valign="middle"><em>22</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> <cell align="left" valign="middle"></cell> @@ -1999,8 +2001,8 @@ <row> <cell align="left" valign="middle"></cell> <cell align="left" valign="middle">MUST support ecdsa_secp256r1_sha256</cell> - <cell align="left" valign="middle"><em>NC</em></cell> - <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22.1</em></cell> </row> <row> diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 4da50d2af8..f4a91cac52 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -973,15 +973,25 @@ scheme_to_components(ecdsa_sha1) -> {sha1, ecdsa, undefined}; scheme_to_components({Hash,Sign}) -> {Hash, Sign, undefined}. -%% TODO: Add support for EC and RSA-SSA signatures -signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha1WithRSAEncryption}) -> - rsa_pkcs1_sha1; +%% TODO: Add support for ed25519, ed448, rsa_pss* signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha256WithRSAEncryption}) -> rsa_pkcs1_sha256; signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha384WithRSAEncryption}) -> rsa_pkcs1_sha384; signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha512WithRSAEncryption}) -> - rsa_pkcs1_sha512. + rsa_pkcs1_sha512; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?'ecdsa-with-SHA256'}) -> + ecdsa_secp256r1_sha256; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?'ecdsa-with-SHA384'}) -> + ecdsa_secp384r1_sha384; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?'ecdsa-with-SHA512'}) -> + ecdsa_secp512r1_sha512; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?'sha-1WithRSAEncryption'}) -> + rsa_pkcs1_sha1; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?sha1WithRSAEncryption}) -> + rsa_pkcs1_sha1; +signature_algorithm_to_scheme(#'SignatureAlgorithm'{algorithm = ?'ecdsa-with-SHA1'}) -> + ecdsa_sha1. %% RFC 5246: 6.2.3.2. CBC Block Cipher diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index a0ae51ed0a..49d20b3ec0 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -433,6 +433,15 @@ certificate_entry(DER) -> %% 79 %% 00 %% 0101010101010101010101010101010101010101010101010101010101010101 +sign(THash, Context, HashAlgo, #'ECPrivateKey'{} = PrivateKey) -> + Content = build_content(Context, THash), + try public_key:sign(Content, HashAlgo, PrivateKey) of + Signature -> + {ok, Signature} + catch + error:badarg -> + {error, badarg} + end; sign(THash, Context, HashAlgo, PrivateKey) -> Content = build_content(Context, THash), @@ -450,7 +459,16 @@ sign(THash, Context, HashAlgo, PrivateKey) -> end. -verify(THash, Context, HashAlgo, Signature, PublicKey) -> +verify(THash, Context, HashAlgo, Signature, {?'id-ecPublicKey', PublicKey, PublicKeyParams}) -> + Content = build_content(Context, THash), + try public_key:verify(Content, HashAlgo, Signature, {PublicKey, PublicKeyParams}) of + Result -> + {ok, Result} + catch + error:badarg -> + {error, badarg} + end; +verify(THash, Context, HashAlgo, Signature, {?rsaEncryption, PublicKey, _PubKeyParams}) -> Content = build_content(Context, THash), %% The length of the Salt MUST be equal to the length of the output @@ -1323,11 +1341,6 @@ get_private_key(#key_share_entry{ {_, PrivateKey}}) -> PrivateKey. -%% TODO: implement EC keys -get_public_key({?'rsaEncryption', PublicKey, _}) -> - PublicKey. - - %% X25519, X448 calculate_shared_secret(OthersKey, MyKey, Group) when is_binary(OthersKey) andalso is_binary(MyKey) andalso @@ -1556,13 +1569,11 @@ verify_certificate_verify(#state{ %% Transcript-Hash uses the HKDF hash function defined by the cipher suite. THash = tls_v1:transcript_hash(Context, HKDFAlgo), - PublicKey = get_public_key(PublicKeyInfo), - ContextString = peer_context_string(Role), %% Digital signatures use the hash function defined by the selected signature %% scheme. - case verify(THash, ContextString, HashAlgo, Signature, PublicKey) of + case verify(THash, ContextString, HashAlgo, Signature, PublicKeyInfo) of {ok, true} -> {ok, {State0, wait_finished}}; {ok, false} -> @@ -1761,15 +1772,20 @@ check_cert_sign_algo(SignAlgo, SignHash, _, ClientSignAlgsCert) -> %% DSA keys are not supported by TLS 1.3 select_sign_algo(dsa, _ClientSignAlgs, _ServerSignAlgs) -> {error, {insufficient_security, no_suitable_public_key}}; -%% TODO: Implement support for ECDSA keys! select_sign_algo(_, [], _) -> {error, {insufficient_security, no_suitable_signature_algorithm}}; select_sign_algo(PublicKeyAlgo, [C|ClientSignAlgs], ServerSignAlgs) -> {_, S, _} = ssl_cipher:scheme_to_components(C), %% RSASSA-PKCS1-v1_5 and Legacy algorithms are not defined for use in signed %% TLS handshake messages: filter sha-1 and rsa_pkcs1. + %% + %% RSASSA-PSS RSAE algorithms: If the public key is carried in an X.509 + %% certificate, it MUST use the rsaEncryption OID. + %% RSASSA-PSS PSS algorithms: If the public key is carried in an X.509 certificate, + %% it MUST use the RSASSA-PSS OID. case ((PublicKeyAlgo =:= rsa andalso S =:= rsa_pss_rsae) - orelse (PublicKeyAlgo =:= rsa_pss andalso S =:= rsa_pss_rsae)) + orelse (PublicKeyAlgo =:= rsa_pss andalso S =:= rsa_pss_pss) + orelse (PublicKeyAlgo =:= ecdsa andalso S =:= ecdsa)) andalso lists:member(C, ServerSignAlgs) of true -> diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 0cdfea0ac2..86a0aaf67b 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -278,7 +278,12 @@ tls13_test_group() -> tls13_ssl_server_with_alpn_ssl_client, tls13_ssl_server_with_alpn_ssl_client_empty_alpn, tls13_ssl_server_with_alpn_ssl_client_bad_alpn, - tls13_ssl_server_with_alpn_ssl_client_alpn]. + tls13_ssl_server_with_alpn_ssl_client_alpn, + tls13_ecdsa_ssl_server_openssl_client, + tls13_ecdsa_ssl_server_ssl_client, + tls13_ecdsa_openssl_server_ssl_client, + tls13_ecdsa_client_auth_ssl_server_ssl_client + ]. %%-------------------------------------------------------------------- init_per_suite(Config0) -> @@ -309,10 +314,17 @@ init_per_group(GroupName, Config) when GroupName == basic_tls; GroupName == options; GroupName == basic; GroupName == session; - GroupName == error_handling_tests_tls; - GroupName == tls13_test_group - -> - ssl_test_lib:clean_tls_version(Config); + GroupName == error_handling_tests_tls -> + ssl_test_lib:clean_tls_version(Config); +%% Do not automatically configure TLS version for the 'tlsv1.3' group +init_per_group('tlsv1.3' = GroupName, Config) -> + case ssl_test_lib:sufficient_crypto_support(GroupName) of + true -> + ssl:start(), + Config; + false -> + {skip, "Missing crypto support"} + end; init_per_group(GroupName, Config) -> ssl_test_lib:clean_tls_version(Config), case ssl_test_lib:is_tls_version(GroupName) andalso ssl_test_lib:sufficient_crypto_support(GroupName) of @@ -1305,14 +1317,14 @@ peername(Config) when is_list(Config) -> {options, [{port, 0} | ClientOpts]}]), ClientPort = ssl_test_lib:inet_port(Client), - ServerIp = ssl_test_lib:node_to_hostip(ServerNode), - ClientIp = ssl_test_lib:node_to_hostip(ClientNode), + ServerIp = ssl_test_lib:node_to_hostip(ServerNode, server), + ClientIp = ssl_test_lib:node_to_hostip(ClientNode, client), ServerMsg = {ok, {ClientIp, ClientPort}}, ClientMsg = {ok, {ServerIp, Port}}, - + ct:log("Testcase ~p, Client ~p Server ~p ~n", [self(), Client, Server]), - + ssl_test_lib:check_result(Server, ServerMsg, Client, ClientMsg), ssl_test_lib:close(Server), @@ -1415,10 +1427,10 @@ sockname(Config) when is_list(Config) -> %% so we can only get a ClientIP, ServerIP will always be 0.0.0.0 {0,0,0,0}; _ -> - ssl_test_lib:node_to_hostip(ServerNode) + ssl_test_lib:node_to_hostip(ServerNode, server) end, - ClientIp = ssl_test_lib:node_to_hostip(ClientNode), + ClientIp = ssl_test_lib:node_to_hostip(ClientNode, client), ServerMsg = {ok, {ServerIp, Port}}, ClientMsg = {ok, {ClientIp, ClientPort}}, @@ -6315,6 +6327,135 @@ tls13_ssl_server_with_alpn_ssl_client_alpn(Config) -> ssl_test_lib:close_port(Client). +tls13_ecdsa_ssl_server_openssl_client() -> + [{doc,"Test TLS 1.3 basic connection between ssl server and openssl s_client using ECDSA certificates"}]. + +tls13_ecdsa_ssl_server_openssl_client(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_ecdsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_ecdsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + +tls13_ecdsa_ssl_server_ssl_client() -> + [{doc,"Test TLS 1.3 basic connection between ssl server and ssl client using ECDSA certificates"}]. + +tls13_ecdsa_ssl_server_ssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_ecdsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_ecdsa_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + ClientOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + +tls13_ecdsa_openssl_server_ssl_client() -> + [{doc,"Test TLS 1.3 basic connection between openssl server and ssl client using ECDSA certificates"}]. + +tls13_ecdsa_openssl_server_ssl_client(Config) -> + process_flag(trap_exit, true), + ServerOpts = ssl_test_lib:ssl_options(server_ecdsa_verify_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_ecdsa_opts, Config), + + ClientOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + + {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), + + Data = "From openssl to erlang", + + Port = ssl_test_lib:inet_port(node()), + CertFile = proplists:get_value(certfile, ServerOpts), + CaCertFile = proplists:get_value(cacertfile, ServerOpts), + KeyFile = proplists:get_value(keyfile, ServerOpts), + Exe = "openssl", + Args = ["s_server", "-accept", integer_to_list(Port), + "-tls1_3", + "-cert", CertFile, "-CAfile", CaCertFile, + "-key", KeyFile, "-Verify", "2"], + + OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), + + ssl_test_lib:wait_for_openssl_server(Port, tls), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + erlang_ssl_receive, [Data]}}, + {options, ClientOpts}]), + true = port_command(OpensslPort, Data), + + ssl_test_lib:check_result(Client, ok), + + %% Clean close down! Server needs to be closed first !! + ssl_test_lib:close_port(OpensslPort), + ssl_test_lib:close(Client), + process_flag(trap_exit, false). + + +tls13_ecdsa_client_auth_ssl_server_ssl_client() -> + [{doc,"TLS 1.3: Test client authentication with ECDSA certificates."}]. + +tls13_ecdsa_client_auth_ssl_server_ssl_client(Config) -> + ClientOpts0 = ssl_test_lib:ssl_options(client_ecdsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_ecdsa_opts, Config), + + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}|ServerOpts0], + ClientOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ClientOpts0], + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + %%Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 32fd917937..a081d65200 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -45,9 +45,18 @@ run_where(_, ipv6) -> Host = rpc:call(ServerNode, net_adm, localhost, []), {ClientNode, ServerNode, Host}. -node_to_hostip(Node) -> +node_to_hostip(Node, Role) -> [_ , Host] = string:tokens(atom_to_list(Node), "@"), {ok, Address} = inet:getaddr(Host, inet), + %% Convert client addresses in 127.0.0.0/24 subnet to the atom 'localhost'. + %% This is a workaround for testcase problems caused by the fact that + %% inet:peername/1 and inet:getaddr/2 return different addresses when + %% running on localhost. + normalize_loopback(Address, Role). + +normalize_loopback({127,_,_,_}, client) -> + localhost; +normalize_loopback(Address, _) -> Address. start_server(Args) -> @@ -393,14 +402,16 @@ close(Pid, Timeout) -> exit(Pid, kill) end. -check_result(Server, ServerMsg, Client, ClientMsg) -> +check_result(Server, ServerMsg, Client, ClientMsg) -> + {ClientIP, ClientPort} = get_ip_port(ServerMsg), receive {Server, ServerMsg} -> check_result(Client, ClientMsg); - + %% Workaround to accept local addresses (127.0.0.0/24) + {Server, {ok, {{127,_,_,_}, ClientPort}}} when ClientIP =:= localhost -> + check_result(Client, ClientMsg); {Client, ClientMsg} -> check_result(Server, ServerMsg); - {Port, {data,Debug}} when is_port(Port) -> ct:log("~p:~p~n Openssl ~s~n",[?MODULE,?LINE, Debug]), check_result(Server, ServerMsg, Client, ClientMsg); @@ -413,10 +424,14 @@ check_result(Server, ServerMsg, Client, ClientMsg) -> ct:fail(Reason) end. -check_result(Pid, Msg) -> +check_result(Pid, Msg) -> + {ClientIP, ClientPort} = get_ip_port(Msg), receive {Pid, Msg} -> ok; + %% Workaround to accept local addresses (127.0.0.0/24) + {Pid, {ok, {{127,_,_,_}, ClientPort}}} when ClientIP =:= localhost -> + ok; {Port, {data,Debug}} when is_port(Port) -> ct:log("~p:~p~n Openssl ~s~n",[?MODULE,?LINE, Debug]), check_result(Pid,Msg); @@ -429,6 +444,13 @@ check_result(Pid, Msg) -> ct:fail(Reason) end. + +get_ip_port({ok,{ClientIP, ClientPort}}) -> + {ClientIP, ClientPort}; +get_ip_port(_) -> + {undefined, undefined}. + + check_server_alert(Pid, Alert) -> receive {Pid, {error, {tls_alert, {Alert, STxt}}}} -> diff --git a/lib/stdlib/src/ordsets.erl b/lib/stdlib/src/ordsets.erl index 176047079b..95b83ded8c 100644 --- a/lib/stdlib/src/ordsets.erl +++ b/lib/stdlib/src/ordsets.erl @@ -150,13 +150,8 @@ union(Es1, []) -> Es1. OrdsetList :: [ordset(T)], Ordset :: ordset(T). -union([S1,S2|Ss]) -> - union1(union(S1, S2), Ss); -union([S]) -> S; -union([]) -> []. - -union1(S1, [S2|Ss]) -> union1(union(S1, S2), Ss); -union1(S1, []) -> S1. +union(OrdsetList) -> + lists:umerge(OrdsetList). %% intersection(OrdSet1, OrdSet2) -> OrdSet. %% Return the intersection of OrdSet1 and OrdSet2. diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 726b409d4d..197564b895 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -33,6 +33,8 @@ %%% BIFs +-export([internal_run/4]). + -export([version/0, compile/1, compile/2, run/2, run/3, inspect/2]). -spec version() -> binary(). @@ -100,6 +102,40 @@ run(_, _) -> run(_, _, _) -> erlang:nif_error(undef). +-spec internal_run(Subject, RE, Options, FirstCall) -> {match, Captured} | + match | + nomatch | + {error, ErrType} when + Subject :: iodata() | unicode:charlist(), + RE :: mp() | iodata() | unicode:charlist(), + Options :: [Option], + Option :: anchored | global | notbol | noteol | notempty + | notempty_atstart | report_errors + | {offset, non_neg_integer()} | + {match_limit, non_neg_integer()} | + {match_limit_recursion, non_neg_integer()} | + {newline, NLSpec :: nl_spec()} | + bsr_anycrlf | bsr_unicode | {capture, ValueSpec} | + {capture, ValueSpec, Type} | CompileOpt, + Type :: index | list | binary, + ValueSpec :: all | all_but_first | all_names | first | none | ValueList, + ValueList :: [ValueID], + ValueID :: integer() | string() | atom(), + CompileOpt :: compile_option(), + Captured :: [CaptureData] | [[CaptureData]], + CaptureData :: {integer(), integer()} + | ListConversionData + | binary(), + ListConversionData :: string() + | {error, string(), binary()} + | {incomplete, string(), binary()}, + ErrType :: match_limit | match_limit_recursion | {compile, CompileErr}, + CompileErr :: {ErrString :: string(), Position :: non_neg_integer()}, + FirstCall :: boolean(). + +internal_run(_, _, _, _) -> + erlang:nif_error(undef). + -spec inspect(MP,Item) -> {namelist, [ binary() ]} when MP :: mp(), Item :: namelist. @@ -765,17 +801,17 @@ do_grun(FlatSubject,Subject,Unicode,CRLF,RE,{Options0,NeedClean}) -> try postprocess(loopexec(FlatSubject,RE,InitialOffset, byte_size(FlatSubject), - Unicode,CRLF,StrippedOptions), + Unicode,CRLF,StrippedOptions,true), SelectReturn,ConvertReturn,FlatSubject,Unicode) catch throw:ErrTuple -> ErrTuple end. -loopexec(_,_,X,Y,_,_,_) when X > Y -> +loopexec(_,_,X,Y,_,_,_,_) when X > Y -> {match,[]}; -loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> - case re:run(Subject,RE,[{offset,X}]++Options) of +loopexec(Subject,RE,X,Y,Unicode,CRLF,Options, First) -> + case re:internal_run(Subject,RE,[{offset,X}]++Options,First) of {error, Err} -> throw({error,Err}); nomatch -> @@ -784,11 +820,11 @@ loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> {match,Rest} = case B>0 of true -> - loopexec(Subject,RE,A+B,Y,Unicode,CRLF,Options); + loopexec(Subject,RE,A+B,Y,Unicode,CRLF,Options,false); false -> {match,M} = - case re:run(Subject,RE,[{offset,X},notempty_atstart, - anchored]++Options) of + case re:internal_run(Subject,RE,[{offset,X},notempty_atstart, + anchored]++Options,false) of nomatch -> {match,[]}; {match,Other} -> @@ -801,7 +837,7 @@ loopexec(Subject,RE,X,Y,Unicode,CRLF,Options) -> forward(Subject,A,1,Unicode,CRLF) end, {match,MM} = loopexec(Subject,RE,NewA,Y, - Unicode,CRLF,Options), + Unicode,CRLF,Options,false), case M of [] -> {match,MM}; diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index ecb514e9f3..d7d57941c2 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -108,7 +108,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-10.4","crypto-3.3", + {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-@OTP-15831:OTP-15836@","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/test/lists_SUITE.erl b/lib/stdlib/test/lists_SUITE.erl index 5dab6f6697..c3c54710eb 100644 --- a/lib/stdlib/test/lists_SUITE.erl +++ b/lib/stdlib/test/lists_SUITE.erl @@ -2600,6 +2600,15 @@ subtract(Config) when is_list(Config) -> [1,2,3,4,5,6,7,8,9,9999,10000,20,21,22] = sub(lists:seq(1, 10000)++[20,21,22], lists:seq(10, 9998)), + %% ERL-986; an integer overflow relating to term comparison + %% caused subtraction to be inconsistent. + Ids = [2985095936,47540628,135460048,1266126295,240535295, + 115724671,161800351,4187206564,4178142725,234897063, + 14773162,6662515191,133150693,378034895,1874402262, + 3507611978,22850922,415521280,253360400,71683243], + + [] = id(Ids) -- id(Ids), + %% Floats/integers. [42.0,42.0] = sub([42.0,42,42.0], [42,42,42]), [1,2,3,4,43.0] = sub([1,2,3,4,5,42.0,43.0], [42.0,5]), @@ -2627,6 +2636,8 @@ subtract(Config) when is_list(Config) -> ok. +id(I) -> I. + sub_non_matching(A, B) -> A = sub(A, B). diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index c9ef9da990..06d8fe9255 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -28,7 +28,8 @@ pcre_compile_workspace_overflow/1,re_infinite_loop/1, re_backwards_accented/1,opt_dupnames/1,opt_all_names/1,inspect/1, opt_no_start_optimize/1,opt_never_utf/1,opt_ucp/1, - match_limit/1,sub_binaries/1,copt/1]). + match_limit/1,sub_binaries/1,copt/1,global_unicode_validation/1, + yield_on_subject_validation/1]). -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/file.hrl"). @@ -45,7 +46,8 @@ all() -> pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, inspect, opt_no_start_optimize,opt_never_utf,opt_ucp, - match_limit, sub_binaries, re_version]. + match_limit, sub_binaries, re_version, global_unicode_validation, + yield_on_subject_validation]. groups() -> []. @@ -200,7 +202,58 @@ re_version(_Config) -> {match,[Version]} = re:run(Version,"^[0-9]\\.[0-9]{2} 20[0-9]{2}-[0-9]{2}-[0-9]{2}",[{capture,all,binary}]), ok. +global_unicode_validation(Config) when is_list(Config) -> + %% Test that unicode validation of the subject is not done + %% for every match found... + Bin = binary:copy(<<"abc\n">>,100000), + {TimeAscii, _} = take_time(fun () -> + re:run(Bin, <<"b">>, [global]) + end), + {TimeUnicode, _} = take_time(fun () -> + re:run(Bin, <<"b">>, [unicode,global]) + end), + if TimeAscii == 0; TimeUnicode == 0 -> + {comment, "Not good enough resolution to compare results"}; + true -> + %% The time the operations takes should be in the + %% same order of magnitude. If validation of the + %% whole subject occurs for every match, the unicode + %% variant will take way longer time... + true = TimeUnicode div TimeAscii < 10 + end. + +take_time(Fun) -> + Start = erlang:monotonic_time(nanosecond), + Res = Fun(), + End = erlang:monotonic_time(nanosecond), + {End-Start, Res}. + +yield_on_subject_validation(Config) when is_list(Config) -> + Go = make_ref(), + Bin = binary:copy(<<"abc\n">>,100000), + {P, M} = spawn_opt(fun () -> + receive Go -> ok end, + {match,[{1,1}]} = re:run(Bin, <<"b">>, [unicode]) + end, + [link, monitor]), + 1 = erlang:trace(P, true, [running]), + P ! Go, + N = count_re_run_trap_out(P, M), + true = N >= 5, + ok. +count_re_run_trap_out(P, M) when is_reference(M) -> + receive {'DOWN',M,process,P,normal} -> ok end, + TD = erlang:trace_delivered(P), + receive {trace_delivered, P, TD} -> ok end, + count_re_run_trap_out(P, 0); +count_re_run_trap_out(P, N) when is_integer(N) -> + receive + {trace,P,out,{erlang,re_run_trap,3}} -> + count_re_run_trap_out(P, N+1) + after 0 -> + N + end. %% Test compile options given directly to run. combined_options(Config) when is_list(Config) -> diff --git a/lib/wx/c_src/Makefile.in b/lib/wx/c_src/Makefile.in index 8ec64bea7e..daadf5e785 100644 --- a/lib/wx/c_src/Makefile.in +++ b/lib/wx/c_src/Makefile.in @@ -70,7 +70,7 @@ else RC_FILE = endif -WX_OBJECTS = $(GENERAL_O) $(GENERATED_O) $(RC_FILE) +WX_OBJECTS = $(GENERATED_O) $(GENERAL_O) $(RC_FILE) OBJECTS = $(WX_OBJECTS) $(GL_OBJECTS) @@ -86,7 +86,7 @@ LD = $(CXX) LDFLAGS = @LDFLAGS@ RESCOMP = @WX_RESCOMP@ -ifeq (@WX_HAVE_STATIC_LIBS@,true) +ifeq (@WX_HAVE_STATIC_LIBS@,true) OPT_WX_LIBS = @WX_LIBS_STATIC@ DEBUG_WX_LIBS = @DEBUG_WX_LIBS_STATIC@ else @@ -97,14 +97,16 @@ endif ifeq ($(TYPE),debug) WX_CFLAGS = @DEBUG_WX_CFLAGS@ CFLAGS = @DEBUG_CFLAGS@ -WX_CXX_FLAGS = @DEBUG_WX_CXXFLAGS@ +WX_CXX_FLAGS = @DEBUG_WX_CXXFLAGS@ CXX_FLAGS = @DEBUG_CXXFLAGS@ +CXX_NO_OPT_FLAGS = @DEBUG_CXXFLAGS@ WX_LIBS = $(DEBUG_WX_LIBS) else WX_CFLAGS = @WX_CFLAGS@ CFLAGS = @CFLAGS@ -WX_CXX_FLAGS = @WX_CXXFLAGS@ +WX_CXX_FLAGS = @WX_CXXFLAGS@ CXX_FLAGS = @CXXFLAGS@ +CXX_NO_OPT_FLAGS = @CXXNOOPTFLAGS@ WX_LIBS = $(OPT_WX_LIBS) endif @@ -113,6 +115,7 @@ GL_LIBS = @GL_LIBS@ CC_O = $(V_CC) -c $(CFLAGS) $(WX_CFLAGS) $(COMMON_CFLAGS) OBJC_CC_O = $(OBJC_CC) -c $(CFLAGS) $(OBJC_CFLAGS) $(WX_CFLAGS) $(COMMON_CFLAGS) CXX_O = $(V_CXX) -c $(CXX_FLAGS) $(WX_CXX_FLAGS) $(COMMON_CFLAGS) +CXX_O_NO_OPT = $(V_CXX) -c $(CXX_NO_OPT_FLAGS) $(WX_CXX_FLAGS) $(COMMON_CFLAGS) # Targets @@ -152,6 +155,10 @@ $(SYS_TYPE)/wxe_ps_init.o: wxe_ps_init.c $(V_at)mkdir -p $(SYS_TYPE) $(cc_verbose)$(OBJC_CC_O) $< -o $@ +$(SYS_TYPE)/wxe_funcs.o: gen/wxe_funcs.cpp + $(V_at)mkdir -p $(SYS_TYPE) + $(CXX_O_NO_OPT) $< -o $@ + $(SYS_TYPE)/%.o: gen/%.cpp $(V_at)mkdir -p $(SYS_TYPE) $(CXX_O) $< -o $@ diff --git a/lib/wx/configure.in b/lib/wx/configure.in index dbe237cd74..f35e6cdbd0 100644 --- a/lib/wx/configure.in +++ b/lib/wx/configure.in @@ -30,17 +30,11 @@ if test -f ./CONF_INFO; then fi if test -z "$ERL_TOP" || test ! -d $ERL_TOP ; then - AC_CONFIG_AUX_DIRS(autoconf) - WX_BUILDING_INSIDE_ERLSRC=false + AC_MSG_ERROR([ERL_TOP is not set]) else erl_top=${ERL_TOP} - if test -d $erl_top/erts/autoconf; then - AC_CONFIG_AUX_DIRS($erl_top/erts/autoconf) - WX_BUILDING_INSIDE_ERLSRC=true - else - AC_CONFIG_AUX_DIRS(autoconf) - WX_BUILDING_INSIDE_ERLSRC=false - fi + AC_CONFIG_AUX_DIRS($erl_top/erts/autoconf) + WX_BUILDING_INSIDE_ERLSRC=true fi if test "X$host" != "Xfree_source" -a "X$host" != "Xwin32"; then @@ -67,6 +61,20 @@ AC_PROG_CXX AC_PROG_RANLIB AC_PROG_CPP +AC_LANG_PUSH([C++]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [[#ifndef __cplusplus + #error "broken C++" + #endif]])],, + [CXX=;]) +AC_LANG_POP([C++]) +if test "X$CXX" = X ; then + echo "Can not find C++ compiler" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Can not find C++ compiler]) +fi +WXERL_CAN_BUILD_DRIVER=false + AC_MSG_NOTICE(Building for [$host_os]) WXERL_CAN_BUILD_DRIVER=true @@ -139,13 +147,9 @@ case $host_os in if test X$APPLE_CC = X -o X$APPLE_CXX = X; then AC_MSG_RESULT([no]) dnl Complete failure, we cannot build Cocoa code - if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_ERROR([Can not find compiler to compile Cocoa applications]) - else - echo "Can not find compiler to compile Cocoa applications" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([Can not find compiler to compile Cocoa applications]) - fi + echo "Can not find compiler to compile Cocoa applications" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Can not find compiler to compile Cocoa applications]) WXERL_CAN_BUILD_DRIVER=false else dnl We think we found an Apple compiler and will add @@ -225,11 +229,21 @@ case $host_os in ;; *) DEBUG_CFLAGS="-g -Wall -fPIC $CFLAGS -DDEBUG" - CFLAGS="-g -Wall -O2 -fPIC $CFLAGS -fomit-frame-pointer -fno-strict-aliasing" + CFLAGS="-Wall -fPIC $CFLAGS -fomit-frame-pointer -fno-strict-aliasing" ;; esac -dnl +dnl +dnl Use -O1 -fno-move-loop-invariants for wxe_funcs.cpp to reduce +dnl compilation time +dnl + +if test "x$GCC" = xyes -a X"$host_os" != X"win32" ; then + CXXNOOPT="-O1" + LM_TRY_ENABLE_CFLAG([-fno-move-loop-invariants], [CXXNOOPT]) +fi + +dnl dnl Opengl tests dnl @@ -251,11 +265,13 @@ if test X"$host_os" != X"win32" ; then AC_CHECK_HEADERS([GL/gl.h]) if test X"$ac_cv_header_GL_gl_h" != Xyes ; then AC_MSG_WARN([No OpenGL headers found, wx will NOT be usable]) + echo "No OpenGL headers found, wx will NOT be usable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false CPPFLAGS="$saved_CPPFLAGS" - else + else GL_LIBS="-L/usr/local/lib $GL_LIBS" fi - else + else GL_LIBS="-L/usr/X11R6/lib $GL_LIBS" fi fi @@ -270,6 +286,8 @@ if test X"$host_os" != X"win32" ; then test X"$ac_cv_header_OpenGL_glu_h" != Xyes then AC_MSG_WARN([No GLU headers found, wx will NOT be usable]) + echo "No GLU headers (glu.h) found, wx will NOT be usable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false fi else AC_CHECK_HEADERS([gl/glu.h],[],[],[#include <windows.h>]) @@ -280,47 +298,17 @@ AC_SUBST(GL_LIBS) DEBUG_CXXFLAGS="$CXXFLAGS $DEBUG_CFLAGS $CPPFLAGS" DEBUG_CFLAGS="$DEBUG_CFLAGS $CPPFLAGS $C_ONLY_FLAGS" -CXXFLAGS="$CXXFLAGS $CFLAGS $CPPFLAGS" +CXXNOOPTFLAGS="$CXXFLAGS $CFLAGS $CPPFLAGS $CXXNOOPT" +CXXFLAGS="$CXXFLAGS $CFLAGS $CPPFLAGS" CFLAGS="$CFLAGS $CPPFLAGS $C_ONLY_FLAGS" AC_SUBST(DEBUG_CFLAGS) AC_SUBST(DEBUG_CXXFLAGS) - -if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_CHECKING(for erl) - if test X$ERL != X; then - AC_MSG_RESULT([yes; using $ERL]) - else - type erl >/dev/null 2>&1 - if test $? -eq 0 ; then - ERL=erl - AC_MSG_RESULT([yes]) - else - AC_MSG_ERROR([Cannot find erl in path]) - fi - fi - AC_MSG_CHECKING(for erlc) - if test X$ERLC != X; then - AC_MSG_RESULT([yes; using $ERLC]) - else - type erlc >/dev/null 2>&1 - if test $? -eq 0 ; then - ERLC=erlc - AC_MSG_RESULT([yes]) - else - AC_MSG_ERROR([Cannot find erlc in path]) - fi - fi - ERLANG_ROOT_DIR=`erl -noshell -eval 'io:format("~s~n",[[code:root_dir()]])' -s erlang halt` - AC_MSG_NOTICE(ERL ROOT DIR: [$ERLANG_ROOT_DIR]) - ERLWX_VSN=`grep WX_VSN $srcdir/vsn.mk | sed 's/^.*=[ ]*//'` -else - ERLC=erlc - ERL=erl - ERLANG_ROOT_DIR=$ERL_TOP - AC_SUBST(ERLC) -fi +ERLC=erlc +ERL=erl +ERLANG_ROOT_DIR=$ERL_TOP +AC_SUBST(ERLC) AC_SUBST(WX_BUILDING_INSIDE_ERLSRC) AC_SUBST(ERLANG_ROOT_DIR) @@ -329,7 +317,7 @@ dnl dnl Check for wxwidgets dnl if test "$cross_compiling" = "yes"; then - echo "Cross compilation of the wx driver is not supported yet, wx will NOT be usable" > ./CONF_INFO + echo "Cross compilation of the wx driver is not supported yet, wx will NOT be usable" >> ./CONF_INFO WXERL_CAN_BUILD_DRIVER=false elif test X"$MIXED_CYGWIN_VC" = X"no" -a X"$MIXED_MSYS_VC" = X"no"; then WX_VERSION=`wx-config --version` @@ -398,13 +386,9 @@ define(wx_warn_text,[ wxWidgets version is $reqwx or above.]) if test "$wxWin" != 1; then - if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_ERROR([wx_warn_text]) - else - echo "wxWidgets not found, wx will NOT be usable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([wx_warn_text]) - fi + echo "wxWidgets not found, wx will NOT be usable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([wx_warn_text]) fi else AC_MSG_CHECKING(for wxWidgets in standard locations) @@ -502,13 +486,9 @@ else if test -z "$WX_LIBS_STATIC"; then AC_MSG_RESULT([failed]) - if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_ERROR([Cannot find core lib version for wxWidgets]) - else - echo "No usable wxWidgets not found, wx will not be useable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([Cannot find core lib version for wxWidgets]) - fi + echo "No usable wxWidgets not found, wx will not be useable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Cannot find core lib version for wxWidgets]) fi WX_HAVE_STATIC_LIBS=true AC_SUBST(WX_CFLAGS) @@ -550,8 +530,8 @@ AC_MSG_RESULT($HAVE_GL_SUPPORT) AC_SUBST(HAVE_GL_SUPPORT) if test X"$HAVE_GL_SUPPORT" != X"yes" ; then - echo "wxWidgets don't have gl support, wx will NOT be useable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false + echo "wxWidgets don't have gl support, wx will NOT be useable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false fi dnl Check for GLintptr @@ -610,7 +590,7 @@ dnl AC_CHECK_HEADERS([wx/stc/stc.h], [], [WXERL_CAN_BUILD_DRIVER=false - echo "wxWidgets don't have wxStyledTextControl (stc.h), wx will NOT be useable" > ./CONF_INFO + echo "wxWidgets don't have wxStyledTextControl (stc.h), wx will NOT be useable" >> ./CONF_INFO AC_MSG_WARN([Can not find wx/stc/stc.h $CXXFLAGS]) ], [#ifdef WIN32 @@ -670,9 +650,9 @@ AC_LANG_POP(C++) AC_MSG_RESULT($CAN_LINK_WX) if test X"$CAN_LINK_WX" != X"yes" ; then - echo "Can not link the wx driver, wx will NOT be useable" > ./CONF_INFO - WXERL_CAN_BUILD_DRIVER=false - AC_MSG_WARN([Can not link wx program are all developer packages installed?]) + echo "Can not link the wx driver, wx will NOT be useable" >> ./CONF_INFO + WXERL_CAN_BUILD_DRIVER=false + AC_MSG_WARN([Can not link wx program are all developer packages installed?]) fi fi dnl - if test "$WXERL_CAN_BUILD_DRIVER" != "false" @@ -721,6 +701,7 @@ esac AC_SUBST(SO_EXT) AC_SUBST(RUN_ERL) +AC_SUBST(CXXNOOPTFLAGS) if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then @@ -737,12 +718,6 @@ CONFIG_STATUS=$WXERL_SYS_TYPE/config.status dnl -if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - if test X"$WXERL_CAN_BUILD_DRIVER" != X"true" ; then - AC_MSG_ERROR([Cannot build wxErlang driver, see ./CONF_INFO for information]) - fi -fi - AC_CONFIG_FILES([ config.mk c_src/Makefile @@ -750,20 +725,9 @@ AC_CONFIG_FILES([ AC_OUTPUT -if test X"$WX_BUILDING_INSIDE_ERLSRC" != X"true" ; then - AC_MSG_NOTICE() - AC_MSG_NOTICE(--------------------------------------------------) - AC_MSG_NOTICE(Using erlang compiler: [$ERLC]) - AC_MSG_NOTICE(wxErlang Install in: [$ERLANG_ROOT_DIR/lib/wx-$ERLWX_VSN]) - AC_MSG_NOTICE(Erlang driver in priv/[$WXERL_SYS_TYPE]/wxe_driver[$SO_EXT]) - AC_MSG_NOTICE(--------------------------------------------------) -fi - -if test X"$WX_BUILDING_INSIDE_ERLSRC" = X"true" ; then - CORES=`ls core* 2>/dev/null` - if test X"$CORES" != X"" ; then - echo "Configure dumped core files" > ignore_core_files - fi +CORES=`ls core* 2>/dev/null` +if test X"$CORES" != X"" ; then + echo "Configure dumped core files" > ignore_core_files fi |