diff options
84 files changed, 1551 insertions, 813 deletions
diff --git a/bootstrap/lib/stdlib/ebin/binary.beam b/bootstrap/lib/stdlib/ebin/binary.beam Binary files differindex 85fc484c32..e712319947 100644 --- a/bootstrap/lib/stdlib/ebin/binary.beam +++ b/bootstrap/lib/stdlib/ebin/binary.beam diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 23f0c66429..f2f380ca4e 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -974,6 +974,10 @@ typedef struct { <seealso marker="#enif_make_binary"><c>enif_make_binary</c></seealso>. An allocated (and owned) <c>ErlNifBinary</c> can be kept between NIF calls.</p> + <p>If you do not need to reallocate or keep the data alive across NIF + calls, consider using <seealso marker="#enif_make_new_binary"> + <c>enif_make_new_binary</c></seealso> instead as it will allocate + small binaries on the process heap when possible.</p> <p>Returns <c>true</c> on success, or <c>false</c> if allocation fails.</p> </desc> @@ -3060,9 +3064,10 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { </name> <fsummary>Get the pid of the calling process.</fsummary> <desc> - <p>Initializes the pid variable <c>*pid</c> to represent the - calling process.</p> - <p>Returns <c>pid</c>.</p> + <p>Initializes the <seealso marker="#ErlNifPid"><c>ErlNifPid</c></seealso> + variable at <c>*pid</c> to represent the calling process.</p> + <p>Returns <c>pid</c> if successful, or NULL if <c>caller_env</c> is not + a <seealso marker="#ErlNifEnv">process-bound environment</seealso>.</p> </desc> </func> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index b6ec3e7ed2..42a368cdd8 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -124,7 +124,6 @@ atom big atom bif_return_trap atom bif_timer_server atom binary -atom binary_bin_to_list_trap atom binary_copy_trap atom binary_find_trap atom binary_longest_prefix_trap diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index c0d5e8ce74..0d1166f6ed 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -551,9 +551,6 @@ bif binary:last/1 bif binary:at/2 bif binary:part/2 binary_binary_part_2 bif binary:part/3 binary_binary_part_3 -bif binary:bin_to_list/1 -bif binary:bin_to_list/2 -bif binary:bin_to_list/3 bif binary:list_to_bin/1 bif binary:copy/1 bif binary:copy/2 @@ -696,3 +693,4 @@ bif erlang:iolist_to_iovec/1 bif erts_internal:new_connection/1 bif erts_internal:abort_connection/2 bif erts_internal:map_next/3 +bif ets:whereis/1 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d4db1a4188..95d324d2c1 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -953,7 +953,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { - BIF_RET(BIF_ARG_1); + if (binary_bitsize(BIF_ARG_1) == 0) { + BIF_RET(BIF_ARG_1); + } + BIF_ERROR(BIF_P, BADARG); } return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index e7acea0c5f..b9b70cd8ef 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -336,6 +336,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "BinVHeap: %b64u\n", p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap: %b64u\n", BIN_OLD_VHEAP(p)); + erts_print(to, to_arg, "BinVHeap unused: %b64u\n", + BIN_VHEAP_SZ(p) - p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap unused: %b64u\n", + BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)); erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { @@ -891,6 +897,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), erts_cbprintf(to, to_arg, "** crashed **\n")); } + for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_CPU_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_cpu_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_CPU_RUNQ); + + for (i = 0; i < erts_no_dirty_io_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_IO_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_io_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_IO_RUNQ); #endif diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index b11903a47b..94e0000c8b 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -990,6 +990,9 @@ ctx_to_bin.execute() { Uint hole_size; Uint orig = mb->orig; ErlSubBin* sb = (ErlSubBin *) boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 41c2ae08d3..33bc189182 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -61,8 +61,6 @@ static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); -static Export binary_bin_to_list_trap_export; -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; @@ -86,10 +84,6 @@ void erts_init_bif_binary(void) am_erlang, am_binary_longest_suffix_trap, 3, &binary_longest_suffix_trap); - erts_init_trap_export(&binary_bin_to_list_trap_export, - am_erlang, am_binary_bin_to_list_trap, 3, - &binary_bin_to_list_trap); - erts_init_trap_export(&binary_copy_trap_export, am_erlang, am_binary_copy_trap, 2, &binary_copy_trap); @@ -2440,191 +2434,6 @@ BIF_RETTYPE binary_at_2(BIF_ALIST_2) BIF_ERROR(BIF_P,BADARG); } -#define BIN_TO_LIST_OK 0 -#define BIN_TO_LIST_TRAP 1 -/* No badarg, checked before call */ - -#define BIN_TO_LIST_LOOP_FACTOR 10 - -static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, - Uint start, Sint *lenp, Eterm *termp) -{ - Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ - Uint len = *lenp; - Uint loops; - Eterm *hp; - Eterm term = *termp; - Uint n; - - ASSERT(reds > 0); - - loops = MIN(reds,len); - - BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); - - hp = HAlloc(p,2*loops); - while (loops--) { - --len; - if (bit_offs) { - n = ((((Uint) bytes[start+len]) << bit_offs) | - (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; - } else { - n = bytes[start+len]; - } - - term = CONS(hp,make_small(n),term); - hp +=2; - } - *termp = term; - *lenp = len; - if (len) { - BUMP_ALL_REDS(p); - return BIN_TO_LIST_TRAP; - } - return BIN_TO_LIST_OK; -} - - -static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, - Uint start, Sint len, Eterm sofar) -{ - Eterm *hp; - Eterm blob; - - hp = HAlloc(p,3); - hp[0] = make_pos_bignum_header(2); - hp[1] = start; - hp[2] = (Uint) len; - blob = make_big(hp); - BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); -} - -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) -{ - Eterm *ptr; - Uint start; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = BIF_ARG_3; - - ptr = big_val(BIF_ARG_2); - start = ptr[1]; - len = (Sint) ptr[2]; - - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); -} - -static BIF_RETTYPE binary_bin_to_list_common(Process *p, - Eterm bin, - Eterm epos, - Eterm elen) -{ - Uint pos; - Sint len; - size_t sz; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(bin)) { - goto badarg; - } - if (!term_to_Uint(epos, &pos)) { - goto badarg; - } - if (!term_to_Sint(elen, &len)) { - goto badarg; - } - if (len < 0) { - Uint lentmp = -(Uint)len; - /* overflow */ - if ((Sint)lentmp < 0) { - goto badarg; - } - len = lentmp; - if (len > pos) { - goto badarg; - } - pos -= len; - } - /* overflow */ - if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { - goto badarg; - } - sz = binary_size(bin); - - if (pos+len > sz) { - goto badarg; - } - ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(p,bin,pos,len,res); - - badarg: - BIF_ERROR(p,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) -{ - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); -} - -BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) -{ - Eterm *tp; - - if (is_not_tuple(BIF_ARG_2)) { - goto badarg; - } - tp = tuple_val(BIF_ARG_2); - if (arityval(*tp) != 2) { - goto badarg; - } - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) -{ - Uint pos = 0; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(BIF_ARG_1)) { - goto badarg; - } - len = binary_size(BIF_ARG_1); - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 9f0c90ff7b..b184adedee 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -639,6 +639,27 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_RET(res); } +Eterm erts_port_data_read(Port* prt) +{ + Eterm res; + erts_aint_t data; + + data = erts_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + return am_undefined; /* Port terminated by racing thread */ + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + res = pdhp->data; + } + return res; +} + + /* * Open a port. Most of the work is not done here but rather in * the file io.c. diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3ba0886464..1ab1c4a363 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1753,6 +1753,28 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** Retrieves the tid() of a named ets table. +*/ +BIF_RETTYPE ets_whereis_1(BIF_ALIST_1) +{ + DbTable* tb; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { + BIF_RET(am_undefined); + } + + res = make_tid(BIF_P, tb); + db_unlock(tb, LCK_READ); + + BIF_RET(res); +} + /* ** The lookup BIF */ @@ -3126,7 +3148,8 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, am_write_concurrency, - am_read_concurrency}; + am_read_concurrency, + am_id}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -4016,7 +4039,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = is_table_named(tb) ? am_true : am_false; } else if (What == am_compressed) { ret = tb->common.compress ? am_true : am_false; + } else if (What == am_id) { + ret = make_tid(p, tb); } + /* * For debugging purposes */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a5a59c8e74..1c64644efc 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -116,6 +116,7 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); static Eterm *full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -142,7 +143,7 @@ static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* heap, Eterm* htop, Eterm* objv, int nobj); + Eterm* htop); static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); @@ -917,6 +918,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc) htop = heap; htop = full_sweep_heaps(p, + ERTS_INVALID_HFRAG_PTR, 1, heap, htop, @@ -1474,18 +1476,22 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); + n = setup_rootset(p, objv, nobj, &rootset); + roots = rootset.roots; + + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { /* * Move heap frags that we know are completely live * directly into the new young heap generation. */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); } - n = setup_rootset(p, objv, nobj, &rootset); - roots = rootset.roots; - GENSWEEP_NSTACK(p, old_htop, n_htop); while (n--) { Eterm* g_ptr = roots->v; @@ -1722,16 +1728,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { - /* - * Move heap frags that we know are completely live - * directly into the heap. - */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); - } - - n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size, + objv, nobj); /* Move the stack to the end of the heap */ stk_sz = HEAP_END(p) - p->stop; @@ -1778,6 +1776,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, static Eterm * full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -1794,6 +1793,19 @@ full_sweep_heaps(Process *p, n = setup_rootset(p, objv, nobj, &rootset); + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); + } + #ifdef HIPE if (hibernate) hipe_empty_nstack(p); @@ -2296,9 +2308,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop) { ErlHeapFragment* qb; char* frag_begin; diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c index c93b5248d9..d779d1031a 100644 --- a/erts/emulator/beam/erl_io_queue.c +++ b/erts/emulator/beam/erl_io_queue.c @@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, ERTS_GET_REAL_BIN(bin_term, orig_pb_term, byte_offset, bit_offset, bit_size); - (void)bit_offset; - (void)bit_size; + ASSERT(bit_size == 0); sb->thing_word = HEADER_SUB_BIN; + sb->bitoffs = bit_offset; sb->bitsize = 0; - sb->bitoffs = 0; sb->orig = orig_pb_term; sb->is_writable = 0; @@ -975,7 +974,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { parent_header = binary_val(parent_binary); binary_size = binary_size(bin_term); - if (bit_offset != 0 || bit_size != 0) { + if (bit_size != 0) { return 0; } else if (binary_size == 0) { state->bytereds_spent += 1; @@ -1017,8 +1016,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { * then just copy it into the accumulator. */ iol2v_expand_acc(state, binary_size); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, binary_size); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + binary_size * 8); + } state->acc_size += binary_size; } else { @@ -1029,8 +1036,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { iol2v_expand_acc(state, spill); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, spill); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + spill * 8); + } state->acc_size += spill; @@ -1179,7 +1194,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { if (is_nil(BIF_ARG_1)) { BIF_RET(NIL); } else if (is_binary(BIF_ARG_1)) { - if (binary_size(BIF_ARG_1) != 0) { + if (binary_bitsize(BIF_ARG_1) != 0) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else if (binary_size(BIF_ARG_1) != 0) { Eterm *hp = HAlloc(BIF_P, 2); BIF_RET(CONS(hp, BIF_ARG_1, NIL)); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 94ebf88b56..d2000aa71e 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -237,9 +237,11 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -/* Temporary object header, auto-deallocated when NIF returns - * or when independent environment is cleared. - */ +/* Temporary object header, auto-deallocated when NIF returns or when + * independent environment is cleared. + * + * The payload can be accessed with &tmp_obj_ptr[1] but keep in mind that its + * first element must not require greater alignment than `next`. */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); @@ -256,6 +258,46 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) } } +/* Whether the given environment is bound to a process and will be cleaned up + * when the NIF returns. It's safe to use temp_alloc for objects in + * env->tmp_obj_list when this is true. */ +static ERTS_INLINE int is_proc_bound(ErlNifEnv *env) +{ + return env->mod_nif != NULL; +} + +/* Allocates and attaches an object to the given environment, running its + * destructor when the environment is cleared. To avoid temporary variables the + * address of the allocated object is returned instead of the enif_tmp_obj_t. + * + * The destructor *must* call `erts_free(tmp_obj->allocator, tmp_obj)` to free + * the object. If the destructor needs to refer to the allocated object its + * address will be &tmp_obj[1]. */ +static ERTS_INLINE void *alloc_tmp_obj(ErlNifEnv *env, size_t size, + void (*dtor)(struct enif_tmp_obj_t*)) { + struct enif_tmp_obj_t *tmp_obj; + ErtsAlcType_t allocator; + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + + tmp_obj = erts_alloc(allocator, sizeof(struct enif_tmp_obj_t) + MAX(1, size)); + + tmp_obj->next = env->tmp_obj_list; + tmp_obj->allocator = allocator; + tmp_obj->dtor = dtor; + + env->tmp_obj_list = tmp_obj; + + return (void*)&tmp_obj[1]; +} + +/* Generic destructor for objects allocated through alloc_tmp_obj that don't + * care about their payload. */ +static void tmp_alloc_dtor(struct enif_tmp_obj_t *tmp_obj) +{ + erts_free(tmp_obj->allocator, tmp_obj); +} + void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); @@ -446,6 +488,7 @@ static void cache_env(ErlNifEnv* env) env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } + void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; @@ -1019,11 +1062,6 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) return is_number(term); } -static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) -{ - return env->mod_nif != NULL; -} - static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); @@ -1058,22 +1096,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; } - bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) -{ - erts_free(obj->allocator, obj); -} - int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { - struct enif_tmp_obj_t* tobj; - ErtsAlcType_t allocator; ErlDrvSizeT sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -1081,7 +1111,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (is_nil(term)) { bin->data = (unsigned char*) &bin->data; /* dummy non-NULL */ bin->size = 0; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; return 1; } @@ -1089,16 +1118,8 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) return 0; } - allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); - tobj->allocator = allocator; - tobj->next = env->tmp_obj_list; - tobj->dtor = &tmp_alloc_dtor; - env->tmp_obj_list = tobj; - - bin->data = (unsigned char*) &tobj[1]; + bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor); bin->size = sz; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; erts_iolist_to_buf(term, (char*) bin->data, sz); ADD_READONLY_CHECK(env, bin->data, bin->size); @@ -1116,7 +1137,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = refbin; return 1; } @@ -1150,12 +1170,10 @@ void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; - ASSERT(bin->bin_term == THE_NON_VALUE); erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; #endif } @@ -1312,29 +1330,51 @@ int enif_get_string(ErlNifEnv *env, ERL_NIF_TERM list, char* buf, unsigned len, Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { - if (bin->bin_term != THE_NON_VALUE) { - return bin->bin_term; - } - else if (bin->ref_bin != NULL) { - Binary* bptr = bin->ref_bin; - Eterm bin_term; - - bin_term = erts_build_proc_bin(&MSO(env->proc), - alloc_heap(env, PROC_BIN_SIZE), - bptr); - if (erts_refc_read(&bptr->intern.refc, 1) == 1) { - /* Total ownership transfer */ - bin->ref_bin = NULL; - bin->bin_term = bin_term; - } - return bin_term; - } - else { - flush_env(env); - bin->bin_term = new_binary(env->proc, bin->data, bin->size); - cache_env(env); - return bin->bin_term; + Eterm bin_term; + + if (bin->ref_bin != NULL) { + Binary* binary = bin->ref_bin; + + /* If the binary is smaller than the heap binary limit we'll return a + * heap binary to reduce the number of small refc binaries in the + * system. We can't simply release the refc binary right away however; + * the documentation states that the binary should be considered + * read-only from this point on, which implies that it should still be + * readable. + * + * We could keep it alive until we return by adding it to the temporary + * object list, but that requires an off-heap allocation which is + * potentially quite slow, so we create a dummy ProcBin instead and + * rely on the next minor GC to get rid of it. */ + if (bin->size <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + + hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(bin->size)); + hb->thing_word = header_heap_bin(bin->size); + hb->size = bin->size; + + sys_memcpy(hb->data, bin->data, bin->size); + + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + + bin_term = make_binary(hb); + } else { + bin_term = erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + } + + /* Our (possibly shared) ownership has been transferred to the term. */ + bin->ref_bin = NULL; + } else { + flush_env(env); + bin_term = new_binary(env->proc, bin->data, bin->size); + cache_env(env); } + + return bin_term; } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, @@ -3284,8 +3324,8 @@ typedef struct { Eterm sublist_start; Eterm sublist_end; - UWord offheap_size; - UWord onheap_size; + UWord referenced_size; + UWord copied_size; UWord iovec_len; } iovec_slice_t; @@ -3295,16 +3335,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul result->sublist_start = list; result->sublist_length = 0; - result->offheap_size = 0; - result->onheap_size = 0; + result->referenced_size = 0; + result->copied_size = 0; result->iovec_len = 0; lookahead = result->sublist_start; while (is_list(lookahead)) { - Eterm *binary_header, binary; + UWord byte_size; + Eterm binary; Eterm *cell; - UWord size; cell = list_val(lookahead); binary = CAR(cell); @@ -3313,35 +3353,36 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 0; } - size = binary_size(binary); - binary_header = binary_val(binary); + byte_size = binary_size(binary); + + if (byte_size > 0) { + int bit_offset, bit_size; + Eterm parent_binary; + UWord byte_offset; - if (size > 0) { - /* If we're a sub-binary we'll need to check our underlying binary - * to determine whether we're on-heap or not. */ - if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { - ErlSubBin *sb = (ErlSubBin*)binary_header; + int requires_copying; - /* Reject bitstrings */ - if((sb->bitoffs + sb->bitsize) > 0) { - return 0; - } + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, + bit_offset, bit_size); - ASSERT(size <= binary_size(sb->orig)); - binary_header = binary_val(sb->orig); + (void)byte_offset; + + if (bit_size != 0) { + return 0; } - if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { - ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + /* If we're unaligned or an on-heap binary we'll need to copy + * ourselves over to a temporary buffer. */ + requires_copying = (bit_offset != 0) || + thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG; - result->iovec_len += 1; - result->onheap_size += size; + if (requires_copying) { + result->copied_size += byte_size; } else { - ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); - - result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; - result->offheap_size += size; + result->referenced_size += byte_size; } + + result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN; } result->sublist_length += 1; @@ -3361,7 +3402,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 1; } -static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { +static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer, + UWord *copy_offset, ErlNifBinary *result) { + Eterm *parent_header; Eterm parent_binary; @@ -3372,10 +3415,11 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + parent_header = binary_val(parent_binary); result->size = binary_size(binary); - result->bin_term = binary; if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { ProcBin *pb = (ProcBin*)parent_header; @@ -3398,24 +3442,48 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { result->data = &((unsigned char*)&hb->data)[byte_offset]; result->ref_bin = NULL; } + + /* If this isn't an *aligned* refc binary, copy its contents to the buffer + * and reference that instead. */ + + if (result->ref_bin == NULL || bit_offset != 0) { + ASSERT(copy_buffer->ref_bin != NULL && copy_buffer->data != NULL); + ASSERT(result->size <= (copy_buffer->size - *copy_offset)); + + if (bit_offset == 0) { + sys_memcpy(©_buffer->data[*copy_offset], + result->data, result->size); + } else { + erts_copy_bits(result->data, bit_offset, 1, + (byte*)©_buffer->data[*copy_offset], 0, 1, + result->size * 8); + } + + result->data = ©_buffer->data[*copy_offset]; + result->ref_bin = copy_buffer->ref_bin; + + *copy_offset += result->size; + } } static int fill_iovec_with_slice(ErlNifEnv *env, iovec_slice_t *slice, ErlNifIOVec *iovec) { - UWord onheap_offset, iovec_idx; - ErlNifBinary onheap_data; + ErlNifBinary copy_buffer = {0}; + UWord copy_offset, iovec_idx; Eterm sublist_iterator; - /* Set up a common refc binary for all on-heap binaries. */ - if (slice->onheap_size > 0) { - if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + /* Set up a common refc binary for all on-heap and unaligned binaries. */ + if (slice->copied_size > 0) { + if (!enif_alloc_binary(slice->copied_size, ©_buffer)) { return 0; } + + ASSERT(copy_buffer.ref_bin != NULL); } sublist_iterator = slice->sublist_start; - onheap_offset = 0; + copy_offset = 0; iovec_idx = 0; while (sublist_iterator != slice->sublist_end) { @@ -3423,20 +3491,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env, Eterm *cell; cell = list_val(sublist_iterator); - inspect_raw_binary_data(CAR(cell), &raw_data); - - /* If this isn't a refc binary, copy its contents to the onheap buffer - * and reference that instead. */ - if (raw_data.size > 0 && raw_data.ref_bin == NULL) { - ASSERT(onheap_offset < onheap_data.size); - ASSERT(slice->onheap_size > 0); - - sys_memcpy(&onheap_data.data[onheap_offset], - raw_data.data, raw_data.size); - - raw_data.data = &onheap_data.data[onheap_offset]; - raw_data.ref_bin = onheap_data.ref_bin; - } + marshal_iovec_binary(CAR(cell), ©_buffer, ©_offset, &raw_data); while (raw_data.size > 0) { UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); @@ -3467,16 +3522,18 @@ static int fill_iovec_with_slice(ErlNifEnv *env, erts_refc_inc(&refc_binary->intern.refc, 1); } - if (slice->onheap_size > 0) { + if (slice->copied_size > 0) { /* Transfer ownership to the iovec; we've taken references to it in * the above loop. */ - enif_release_binary(&onheap_data); + enif_release_binary(©_buffer); } } else { - if (slice->onheap_size > 0) { - /* Attach the binary to our environment and let the GC take care of - * it after returning. */ - enif_make_binary(env, &onheap_data); + if (slice->copied_size > 0) { + /* Attach the binary to our environment and let the next minor GC + * get rid of it. This is slightly faster than using the tmp object + * list since it avoids off-heap allocations. */ + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), copy_buffer.ref_bin); } } @@ -3502,19 +3559,14 @@ static int create_iovec_from_slice(ErlNifEnv *env, alloc_size = binv_offset; alloc_size += slice->iovec_len * sizeof(Binary*); - /* If we have an environment we'll attach the allocated data to it. The - * GC will take care of releasing it later on. */ + /* When the user passes an environment, we attach the iovec to it so + * the user won't have to bother managing it (similar to + * enif_inspect_binary). It'll disappear once the environment is + * cleaned up. */ if (env != NULL) { - ErlNifBinary gc_bin; - - if (!enif_alloc_binary(alloc_size, &gc_bin)) { - return 0; - } - - alloc_base = (char*)gc_bin.data; - enif_make_binary(env, &gc_bin); + alloc_base = alloc_tmp_obj(env, alloc_size, &tmp_alloc_dtor); } else { - alloc_base = enif_alloc(alloc_size); + alloc_base = erts_alloc(ERTS_ALC_T_NIF, alloc_size); } iovec = (ErlNifIOVec*)alloc_base; @@ -3523,12 +3575,12 @@ static int create_iovec_from_slice(ErlNifEnv *env, iovec->flags = 0; } - iovec->size = slice->offheap_size + slice->onheap_size; + iovec->size = slice->referenced_size + slice->copied_size; iovec->iovcnt = slice->iovec_len; if(!fill_iovec_with_slice(env, slice, iovec)) { if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { - enif_free(iovec); + erts_free(ERTS_ALC_T_NIF, iovec); } return 0; @@ -4218,34 +4270,31 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size); struct readonly_check_t { - struct enif_tmp_obj_t hdr; unsigned char* ptr; unsigned size; unsigned checksum; }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - struct readonly_check_t* obj = erts_alloc(allocator, - sizeof(struct readonly_check_t)); - obj->hdr.allocator = allocator; - obj->hdr.next = env->tmp_obj_list; - env->tmp_obj_list = &obj->hdr; - obj->hdr.dtor = &readonly_check_dtor; + struct readonly_check_t* obj; + + obj = alloc_tmp_obj(env, sizeof(struct readonly_check_t), + &readonly_check_dtor); + obj->ptr = ptr; obj->size = sz; - obj->checksum = calc_checksum(ptr, sz); + obj->checksum = calc_checksum(ptr, sz); } -static void readonly_check_dtor(struct enif_tmp_obj_t* o) +static void readonly_check_dtor(struct enif_tmp_obj_t* tmp_obj) { - struct readonly_check_t* obj = (struct readonly_check_t*) o; - unsigned chksum = calc_checksum(obj->ptr, obj->size); - if (chksum != obj->checksum) { + struct readonly_check_t* ro_check = (struct readonly_check_t*)&tmp_obj[1]; + unsigned chksum = calc_checksum(ro_check->ptr, ro_check->size); + if (chksum != ro_check->checksum) { fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" - " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + " %x != %x\r\nABORTING\r\n", chksum, ro_check->checksum); abort(); } - erts_free(obj->hdr.allocator, obj); + erts_free(tmp_obj->allocator, tmp_obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 053f7673c4..a99b4db705 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -137,8 +137,9 @@ typedef struct unsigned char* data; /* Internals (avert your eyes) */ - ERL_NIF_TERM bin_term; void* ref_bin; + /* for future additions to be ABI compatible (same struct size) */ + void* __spare__[2]; }ErlNifBinary; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 9117eb1f72..6b90187d60 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -180,6 +180,7 @@ void erts_init_port_data(Port *); void erts_cleanup_port_data(Port *); Uint erts_port_data_size(Port *); ErlOffHeap *erts_port_data_offheap(Port *); +Eterm erts_port_data_read(Port* prt); #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a807d60ec7..3baca6ce79 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1538,6 +1538,20 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } +void +erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp) +{ + ErtsProcList *first = plp; + + while (plp) { + erts_print(to, to_arg, "%T", plp->pid); + plp = plp->next; + if (plp == first) + break; + } + erts_print(to, to_arg, "\n"); +} + void * erts_psd_set_init(Process *p, int ix, void *data) { @@ -13398,16 +13412,33 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) return yreg; } +static void print_current_process_info(fmtfn_t, void *to_arg, ErtsSchedulerData*); + /* * Print scheduler information */ void -erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) +{ int i; erts_aint32_t flg; - Process *p; - erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + switch (esdp->type) { + case ERTS_SCHED_NORMAL: + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + break; + case ERTS_SCHED_DIRTY_CPU: + erts_print(to, to_arg, "=dirty_cpu_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers)); + break; + case ERTS_SCHED_DIRTY_IO: + erts_print(to, to_arg, "=dirty_io_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers + erts_no_dirty_cpu_schedulers)); + break; + default: + erts_print(to, to_arg, "=unknown_scheduler_type:%u\n", esdp->type); + break; + } flg = erts_atomic32_read_dirty(&esdp->ssi->flags); erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); @@ -13453,10 +13484,24 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } erts_print(to, to_arg, "\n"); - erts_print(to, to_arg, "Current Port: "); - if (esdp->current_port) - erts_print(to, to_arg, "%T", esdp->current_port->common.id); - erts_print(to, to_arg, "\n"); + if (esdp->type == ERTS_SCHED_NORMAL) { + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + erts_print_run_queue_info(to, to_arg, esdp->run_queue); + } + + /* This *MUST* to be the last information in scheduler block */ + print_current_process_info(to, to_arg, esdp); +} + +void erts_print_run_queue_info(fmtfn_t to, void *to_arg, + ErtsRunQueue *run_queue) +{ + erts_aint32_t flg; + int i; for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { erts_print(to, to_arg, "Run Queue "); @@ -13478,12 +13523,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { break; } erts_print(to, to_arg, "Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + erts_atomic32_read_dirty(&run_queue->procs.prio_info[i].len)); } erts_print(to, to_arg, "Run Queue Port Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + erts_atomic32_read_dirty(&run_queue->ports.info.len)); - flg = erts_atomic32_read_dirty(&esdp->run_queue->flags); + flg = erts_atomic32_read_dirty(&run_queue->flags); erts_print(to, to_arg, "Run Queue Flags: "); for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -13550,9 +13595,15 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); +} + + +static void print_current_process_info(fmtfn_t to, void *to_arg, + ErtsSchedulerData* esdp) +{ + Process *p = esdp->current_process; + erts_aint32_t flg; - /* This *MUST* to be the last information in scheduler block */ - p = esdp->current_process; erts_print(to, to_arg, "Current Process: "); if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { flg = erts_atomic32_read_dirty(&p->state); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 55c020d47b..4f200b13ec 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1546,6 +1546,7 @@ Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); +void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *); @@ -1792,6 +1793,7 @@ void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*); void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index f562fc961b..0d76a4ef53 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -78,8 +78,17 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { erts_aint32_t state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) - dump_process_info(to, to_arg, p); + if (state & ERTS_PSFLG_EXITING) + continue; + if (state & ERTS_PSFLG_GC) { + ErtsSchedulerData *sdp = erts_get_scheduler_data(); + if (!sdp || p != sdp->current_process) + continue; + + /* We want to dump the garbing process that caused the dump */ + } + + dump_process_info(to, to_arg, p); } } @@ -135,9 +144,12 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) ErtsMessage* mp; int yreg = -1; + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) + return; + ERTS_MSGQ_MV_INQ2PRIVQ(p); - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { + if (p->msg.first) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); for (mp = p->msg.first; mp != NULL; mp = mp->next) { Eterm mesg = ERL_MESSAGE_TERM(mp); @@ -152,38 +164,34 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) } } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - if (p->dictionary) { - erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); - erts_deep_dictionary_dump(to, to_arg, - p->dictionary, dump_element_nl); - } + if (p->dictionary) { + erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); + erts_deep_dictionary_dump(to, to_arg, + p->dictionary, dump_element_nl); } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, sp, yreg); - } + erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + yreg = stack_element_dump(to, to_arg, sp, yreg); + } - erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - Eterm term = *sp; - - if (!is_catch(term) && !is_CP(term)) { - heap_dump(to, to_arg, term); - } - } - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - heap_dump(to, to_arg, mesg); - mesg = ERL_MESSAGE_TOKEN(mp); - heap_dump(to, to_arg, mesg); - } - if (p->dictionary) { - erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); - } + erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + Eterm term = *sp; + + if (!is_catch(term) && !is_CP(term)) { + heap_dump(to, to_arg, term); + } + } + for (mp = p->msg.first; mp != NULL; mp = mp->next) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + heap_dump(to, to_arg, mesg); + mesg = ERL_MESSAGE_TOKEN(mp); + heap_dump(to, to_arg, mesg); + } + if (p->dictionary) { + erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); } } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4b996d8fc2..d2495479ab 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2783,24 +2783,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, ASSERT(0); } - /* Only remove tracer on self() and ports */ + /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */ if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsProcLocks c_p_xlocks = 0; - if (is_internal_pid(t_p->id)) { - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); - if (c_p_locks != ERTS_PROC_LOCKS_ALL) { - c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; - if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { - erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (is_internal_pid(t_p->id)) { + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + if (c_p_locks != ERTS_PROC_LOCKS_ALL) { + c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; + if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } } } - } - erts_tracer_replace(t_p, erts_tracer_nil); - t_p->trace_flags &= ~TRACEE_FLAGS; - if (c_p_xlocks) - erts_proc_unlock(c_p, c_p_xlocks); + erts_tracer_replace(t_p, erts_tracer_nil); + t_p->trace_flags &= ~TRACEE_FLAGS; + + if (c_p_xlocks) + erts_proc_unlock(c_p, c_p_xlocks); + } } return 0; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2c1b7871c4..4bb2c0cb01 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -5075,6 +5075,93 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) erts_print(prtd->to, prtd->arg, "%T", lnk->pid); } +static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) +{ + erts_aint32_t rest; + int unknown = 0; + char delim = ' '; + + erts_print(to, arg, "State:"); + + rest = state; + while (rest) { + erts_aint32_t chk = (rest ^ (rest-1)) & rest; /* lowest set bit */ + char* s; + + rest &= ~chk; + switch (chk) { + case ERTS_PORT_SFLG_CONNECTED: s = "CONNECTED"; break; + case ERTS_PORT_SFLG_EXITING: s = "EXITING"; break; + case ERTS_PORT_SFLG_DISTRIBUTION: s = "DISTR"; break; + case ERTS_PORT_SFLG_BINARY_IO: s = "BINARY_IO"; break; + case ERTS_PORT_SFLG_SOFT_EOF: s = "SOFT_EOF"; break; + case ERTS_PORT_SFLG_CLOSING: s = "CLOSING"; break; + case ERTS_PORT_SFLG_SEND_CLOSED: s = "SEND_CLOSED"; break; + case ERTS_PORT_SFLG_LINEBUF_IO: s = "LINEBUF_IO"; break; + case ERTS_PORT_SFLG_FREE: s = "FREE"; break; + case ERTS_PORT_SFLG_INITIALIZING: s = "INITIALIZING"; break; + case ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK: s = "PORT_LOCK"; break; + case ERTS_PORT_SFLG_INVALID: s = "INVALID"; break; + case ERTS_PORT_SFLG_HALT: s = "HALT"; break; +#ifdef DEBUG + case ERTS_PORT_SFLG_PORT_DEBUG: s = "DEBUG"; break; +#endif + default: + unknown = 1; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown || !state) + erts_print(to, arg, "%c0x%x\n", delim, state); + else + erts_print(to, arg, "\n"); +} + +static void dump_port_task_flags(fmtfn_t to, void *arg, Port* p) +{ + erts_aint32_t flags = erts_atomic32_read_nob(&p->sched.flags); + erts_aint32_t unknown = 0; + char delim = ' '; + + if (!flags) + return; + + erts_print(to, arg, "Task Flags:"); + + while (flags) { + erts_aint32_t chk = (flags ^ (flags-1)) & flags; /* lowest set bit */ + char* s; + + flags &= ~chk; + switch (chk) { + case ERTS_PTS_FLG_IN_RUNQ: s = "IN_RUNQ"; break; + case ERTS_PTS_FLG_EXEC: s = "EXEC"; break; + case ERTS_PTS_FLG_HAVE_TASKS: s = "HAVE_TASKS"; break; + case ERTS_PTS_FLG_EXIT: s = "EXIT"; break; + case ERTS_PTS_FLG_BUSY_PORT: s = "BUSY_PORT"; break; + case ERTS_PTS_FLG_BUSY_PORT_Q: s = "BUSY_Q"; break; + case ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q: s = "CHK_UNSET_BUSY_Q"; break; + case ERTS_PTS_FLG_HAVE_BUSY_TASKS: s = "BUSY_TASKS"; break; + case ERTS_PTS_FLG_HAVE_NS_TASKS: s = "NS_TASKS"; break; + case ERTS_PTS_FLG_PARALLELISM: s = "PARALLELISM"; break; + case ERTS_PTS_FLG_FORCE_SCHED: s = "FORCE_SCHED"; break; + case ERTS_PTS_FLG_EXITING: s = "EXITING"; break; + case ERTS_PTS_FLG_EXEC_IMM: s = "EXEC_IMM"; break; + default: + unknown |= chk; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown) + erts_print(to, arg, "%cUNKNOWN(0x%x)\n", delim, unknown); + else + erts_print(to, arg, "\n"); +} + void print_port_info(Port *p, fmtfn_t to, void *arg) { @@ -5084,6 +5171,8 @@ print_port_info(Port *p, fmtfn_t to, void *arg) return; erts_print(to, arg, "=port:%T\n", p->common.id); + dump_port_state(to, arg, state); + dump_port_task_flags(to, arg, p); erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id)); if (state & ERTS_PORT_SFLG_CONNECTED) { erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p)); @@ -5106,6 +5195,10 @@ print_port_info(Port *p, fmtfn_t to, void *arg) erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); erts_print(to, arg, "\n"); } + if (p->suspended) { + erts_print(to, arg, "Suspended: "); + erts_proclist_dump(to, arg, p->suspended); + } if (p->common.u.alive.reg != NULL) erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name); @@ -5123,6 +5216,14 @@ print_port_info(Port *p, fmtfn_t to, void *arg) } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } + erts_print(to, arg, "Input: %beu\n", p->bytes_in); + erts_print(to, arg, "Output: %beu\n", p->bytes_out); + erts_print(to, arg, "Queue: %beu\n", erts_ioq_size(&p->ioq)); + { + Eterm port_data = erts_port_data_read(p); + if (port_data != am_undefined) + erts_print(to, arg, "Port Data: %T\n", port_data); + } } void diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index fe9f1c7606..4bf60619ba 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3152,6 +3152,9 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; + if (eq_only && a_size != b_size) { + RETURN_NEQ(a_size - b_size); + } ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c index 57c8ef62e1..4a6c476882 100644 --- a/erts/emulator/nifs/unix/unix_prim_file.c +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -125,24 +125,11 @@ static int open_file_type_check(const efile_path_t *path, int fd) { * immediately in a read within the call, but the new implementation * never does that. */ return 1; - } else { - /* The old driver tolerated opening /dev/null despite the "no devices" - * limitation. It provided no explanation for this but we still need - * to match the behavior. We're checking through stat(2) instead of - * comparing the name to account for links. */ - struct stat null_device_info; - int is_dev_null; - - is_dev_null = (stat("/dev/null", &null_device_info) == 0); - is_dev_null &= (file_info.st_ino == null_device_info.st_ino); - is_dev_null &= (file_info.st_dev == null_device_info.st_dev); - - if(is_dev_null) { - return 1; - } } - if(!S_ISREG(file_info.st_mode)) { + /* Allow everything that isn't a directory, and error out on the next call + * if it's unsupported. */ + if(S_ISDIR(file_info.st_mode)) { return 0; } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 4bc1838139..a3c3daac15 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -262,6 +262,7 @@ test_deep_bitstr(List) -> {Bin,bitstring_to_list(Bin)}. bad_list_to_binary(Config) when is_list(Config) -> + test_bad_bin(<<1:1>>), test_bad_bin(atom), test_bad_bin(42), test_bad_bin([1|2]), diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl index 49dc64b0d2..963b7e2501 100644 --- a/erts/emulator/test/iovec_SUITE.erl +++ b/erts/emulator/test/iovec_SUITE.erl @@ -25,7 +25,8 @@ -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, sub_binary_lists/1, iolist_to_iovec_idempotence/1, - iolist_to_iovec_correctness/1]). + iolist_to_iovec_correctness/1, unaligned_sub_binaries/1, + direct_binary_arg/1]). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +37,8 @@ suite() -> all() -> [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists, sub_binary_lists, illegal_lists, improper_lists, cons_bomb, - iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. + iolist_to_iovec_idempotence, iolist_to_iovec_correctness, + unaligned_sub_binaries, direct_binary_arg]. init_per_suite(Config) -> Config. @@ -78,7 +80,7 @@ illegal_lists(Config) when is_list(Config) -> BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]), BadInts = gen_variations(["gurka", 890, "gaffel"]), Atoms = gen_variations([gurka, "gaffel"]), - BadTails = [["test" | 0], ["gurka", gaffel]], + BadTails = [["test" | 0], ["gurka" | gaffel], ["gaffel" | <<1:1>>]], Variations = BitStrs ++ BadInts ++ Atoms ++ BadTails, @@ -98,14 +100,7 @@ cons_bomb(Config) when is_list(Config) -> BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), - Rounds = - case system_mem_size() of - Mem when Mem >= (16 bsl 30) -> 32; - Mem when Mem >= (3 bsl 30) -> 28; - _ -> 20 - end, - - Variations = gen_variations([IntBase, BinBase, MixBase], Rounds), + Variations = gen_variations([IntBase, BinBase, MixBase], 16), equivalence_test(fun erlang:iolist_to_iovec/1, Variations). iolist_to_iovec_idempotence(Config) when is_list(Config) -> @@ -130,6 +125,21 @@ iolist_to_iovec_correctness(Config) when is_list(Config) -> true = is_iolist_equal(Optimized, Variations), ok. +unaligned_sub_binaries(Config) when is_list(Config) -> + UnalignedBins = [gen_unaligned_binary(I) || I <- lists:seq(32, 4 bsl 10, 512)], + UnalignedVariations = gen_variations(UnalignedBins), + + Optimized = erlang:iolist_to_iovec(UnalignedVariations), + + true = is_iolist_equal(Optimized, UnalignedVariations), + ok. + +direct_binary_arg(Config) when is_list(Config) -> + {'EXIT',{badarg, _}} = (catch erlang:iolist_to_iovec(<<1:1>>)), + [<<1>>] = erlang:iolist_to_iovec(<<1>>), + [] = erlang:iolist_to_iovec(<<>>), + ok. + illegality_test(Fun, Variations) -> [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations], ok. @@ -145,11 +155,18 @@ equivalence_test(Fun, [Head | _] = Variations) -> is_iolist_equal(A, B) -> iolist_to_binary(A) =:= iolist_to_binary(B). +gen_unaligned_binary(Size) -> + Bin0 = << <<I>> || I <- lists:seq(1, Size) >>, + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% Generates a bunch of lists whose contents will be equal to Base repeated a %% few times. The lists only differ by their structure, so their reduction to %% a simpler format should yield the same result. gen_variations(Base) -> - gen_variations(Base, 16). + gen_variations(Base, 12). gen_variations(Base, N) -> [gen_flat_list(Base, N), gen_nested_list(Base, N), @@ -169,8 +186,3 @@ gen_nasty_list_1([Head | Base], Result) when is_list(Head) -> gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]); gen_nasty_list_1([Head | Base], Result) -> gen_nasty_list_1(Base, [[Result], [Head]]). - -system_mem_size() -> - application:ensure_all_started(os_mon), - {Tot,_Used,_} = memsup:get_memory_data(), - Tot. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 85c302e310..b6e15ababb 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -3049,8 +3049,24 @@ nif_ioq(Config) -> {peek_head, a}, {peek, a}, - %% Test to enqueue a bunch of refc binaries + %% Ensure that enqueued refc binaries are intact after a roundtrip. + %% + %% This test and the ones immediately following it does not go through + %% erlang:iolist_to_iovec/1 {enqv, a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... heap binaries + {enqv, a, [nif_ioq_payload(heapbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... plain sub-binaries + {enqv, a, [nif_ioq_payload(subbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... unaligned binaries + {enqv, a, [nif_ioq_payload(unaligned_bin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, %% Enq stuff to destroy with data in queue {enqv, a, 2, 100}, @@ -3206,13 +3222,16 @@ nif_ioq_payload(N) when is_integer(N) -> Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end, Head = element(1, lists:split(N,[nif_ioq_payload(subbin), nif_ioq_payload(heapbin), - nif_ioq_payload(refcbin) | Tail])), + nif_ioq_payload(refcbin), + nif_ioq_payload(unaligned_bin) | Tail])), erlang:iolist_to_iovec(Head); nif_ioq_payload(subbin) -> Bin = nif_ioq_payload(refcbin), Sz = size(Bin) - 1, <<_:8,SubBin:Sz/binary,_/bits>> = Bin, SubBin; +nif_ioq_payload(unaligned_bin) -> + make_unaligned_binary(<< <<I>> || I <- lists:seq(1, 255) >>); nif_ioq_payload(heapbin) -> <<"a literal heap binary">>; nif_ioq_payload(refcbin) -> @@ -3220,6 +3239,13 @@ nif_ioq_payload(refcbin) -> nif_ioq_payload(Else) -> Else. +make_unaligned_binary(Bin0) -> + Size = byte_size(Bin0), + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% The NIFs: lib_version() -> undefined. call_history() -> ?nif_stub. diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 2d86dd3843..02e77bfbb2 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl index d16fe5a0bc..b82b9f5b46 100644 --- a/erts/preloaded/src/erl_prim_loader.erl +++ b/erts/preloaded/src/erl_prim_loader.erl @@ -56,7 +56,7 @@ -export([purge_archive_cache/0]). %% Used by init and the code server. --export([get_modules/2,get_modules/3]). +-export([get_modules/2,get_modules/3, is_basename/1]). -include_lib("kernel/include/file.hrl"). diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index b3d0ba20bc..bb1afecafc 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -40,7 +40,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap, {minutes, 5}}]. + {timetrap, {minutes, 10}}]. all() -> [core_files]. diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 6bd3c2f9a6..c5f93a3392 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -3444,19 +3444,19 @@ state__fun_info(Fun, #state{callgraph = CG, fun_tab = FunTab, plt = PLT}) -> {Fun, Sig, Contract, LocalRet}. forward_args(Fun, ArgTypes, #state{work = Work, fun_tab = FunTab} = State) -> - {OldArgTypes, OldOut, Fixpoint} = + {NewArgTypes, OldOut, Fixpoint} = case dict:find(Fun, FunTab) of - {ok, {not_handled, {OldArgTypes0, OldOut0}}} -> - {OldArgTypes0, OldOut0, false}; + {ok, {not_handled, {_OldArgTypesAreNone, OldOut0}}} -> + {ArgTypes, OldOut0, false}; {ok, {OldArgTypes0, OldOut0}} -> - {OldArgTypes0, OldOut0, - t_is_subtype(t_product(ArgTypes), t_product(OldArgTypes0))} + NewArgTypes0 = [t_sup(X, Y) || + {X, Y} <- lists:zip(ArgTypes, OldArgTypes0)], + {NewArgTypes0, OldOut0, + t_is_equal(t_product(NewArgTypes0), t_product(OldArgTypes0))} end, case Fixpoint of true -> State; false -> - NewArgTypes = [t_sup(X, Y) || - {X, Y} <- lists:zip(ArgTypes, OldArgTypes)], NewWork = add_work(Fun, Work), ?debug("~tw: forwarding args ~ts\n", [state__lookup_name(Fun, State), diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 95c8b5ebce..2af4534396 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -531,17 +531,19 @@ compute_md5_from_files(Files) -> lists:keysort(1, [{F, compute_md5_from_file(F)} || F <- Files]). compute_md5_from_file(File) -> - case filelib:is_regular(File) of - false -> + case beam_lib:all_chunks(File) of + {ok, _, Chunks} -> + %% We cannot use beam_lib:md5 because it does not consider + %% the debug_info chunk, where typespecs are likely stored. + %% So we consider almost all chunks except the useless ones. + Filtered = [[ID, Chunk] || {ID, Chunk} <- Chunks, ID =/= "CInf", ID =/= "Docs"], + erlang:md5(lists:sort(Filtered)); + {error, beam_lib, {file_error, _, enoent}} -> Msg = io_lib:format("Not a regular file: ~ts\n", [File]), throw({dialyzer_error, Msg}); - true -> - case dialyzer_utils:get_core_from_beam(File) of - {error, Error} -> - throw({dialyzer_error, Error}); - {ok, Core} -> - erlang:md5(term_to_binary(Core)) - end + {error, beam_lib, _} -> + Msg = io_lib:format("Could not compute MD5 for .beam: ~ts\n", [File]), + throw({dialyzer_error, Msg}) end. init_diff_list(RemoveFiles, AddFiles) -> diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl new file mode 100644 index 0000000000..44149f4199 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same.erl @@ -0,0 +1,15 @@ +-module(same). + +-export([baz/1]). + +-record(bar, { + a :: same_type:st(integer()), + b :: same_type:st(atom()) + }). + +baz(Bar) -> + _ = wrap_find(0, Bar#bar.a), + wrap_find(0, Bar#bar.b). + +wrap_find(K, D) -> + same_type:t(K, D). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl new file mode 100644 index 0000000000..855a5d30be --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para_bug/same_type.erl @@ -0,0 +1,13 @@ +-module(same_type). + +-export([t/2]). + +-export_type([st/1]). + +%% When unopaqued all specializations of st/1 are equal. +-opaque st(_A) :: {st, tuple()}. + +-spec t(_, st(_)) -> _. + +t(K, V) -> + {K, V}. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 1d569bd2ff..e398b4f0aa 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -711,9 +711,9 @@ do_handle_info({'EXIT', _, _}, State = #state{request = undefined}) -> %% can retry requests in the pipeline. do_handle_info({'EXIT', _, _}, State) -> {noreply, State#state{status = close}}. - + call(Msg, Pid) -> - try gen_server:call(Pid, Msg) + try gen_server:call(Pid, Msg, infinity) catch exit:{noproc, _} -> {error, closed}; diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index fdf4cc6e07..e24a4a8694 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -18,13 +18,13 @@ %% %CopyrightEnd% {"%VSN%", [ - {<<"6.4.3">>, [{load_module, httpd_esi, + {<<"6.4.5">>, [{load_module, httpc_handler, soft_purge, soft_purge, []}]}, {<<"6\\..*">>,[{restart_application, inets}]}, {<<"5\\..*">>,[{restart_application, inets}]} ], [ - {<<"6.4.3">>, [{load_module, httpd_esi, + {<<"6.4.5">>, [{load_module, httpc_handler, soft_purge, soft_purge, []}]}, {<<"6\\..*">>,[{restart_application, inets}]}, {<<"5\\..*">>,[{restart_application, inets}]} diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 0533b9ab70..4bb449408f 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -50,6 +50,7 @@ all() -> [ {group, http}, {group, sim_http}, + {group, http_internal}, {group, https}, {group, sim_https}, {group, misc} @@ -62,6 +63,7 @@ groups() -> %% and it shall be the last test case in the suite otherwise cookie %% will fail. {sim_http, [], only_simulated() ++ [process_leak_on_keepalive]}, + {http_internal, [], real_requests_esi()}, {https, [], real_requests()}, {sim_https, [], only_simulated()}, {misc, [], misc()} @@ -97,6 +99,9 @@ real_requests()-> invalid_body ]. +real_requests_esi() -> + [slow_connection]. + only_simulated() -> [ cookie, @@ -1245,7 +1250,25 @@ stream_fun_server_close(Config) when is_list(Config) -> after 13000 -> ct:fail(did_not_receive_close) end. - + +%%-------------------------------------------------------------------- +slow_connection() -> + [{doc, "Test that a request on a slow keep-alive connection won't crash the httpc_manager"}]. +slow_connection(Config) when is_list(Config) -> + BodyFun = fun(0) -> eof; + (LenLeft) -> timer:sleep(1000), + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + Request = {url(group_name(Config), "/httpc_SUITE:esi_post", Config), + [{"content-length", "100"}], + "text/plain", + {BodyFun, 100}}, + {ok, _} = httpc:request(post, Request, [], []), + %% Second request causes a crash if gen_server timeout is not set to infinity + %% in httpc_handler. + {ok, _} = httpc:request(post, Request, [], []). + + %%-------------------------------------------------------------------- %% Internal Functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -1339,6 +1362,8 @@ url(https, End, Config) -> ?TLS_URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; url(sim_http, End, Config) -> url(http, End, Config); +url(http_internal, End, Config) -> + url(http, End, Config); url(sim_https, End, Config) -> url(https, End, Config). url(http, UserInfo, End, Config) -> @@ -1385,7 +1410,17 @@ server_config(http, Config) -> {mime_type, "text/plain"}, {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} ]; - +server_config(http_internal, Config) -> + ServerRoot = proplists:get_value(server_root, Config), + [{port, 0}, + {server_name,"httpc_test"}, + {server_root, ServerRoot}, + {document_root, proplists:get_value(doc_root, Config)}, + {bind_address, any}, + {ipfamily, inet_version()}, + {mime_type, "text/plain"}, + {erl_script_alias, {"", [httpc_SUITE]}} + ]; server_config(https, Config) -> [{socket_type, {essl, ssl_config(Config)}} | server_config(http, Config)]; server_config(sim_https, Config) -> @@ -1393,6 +1428,9 @@ server_config(sim_https, Config) -> server_config(_, _) -> []. +esi_post(Sid, _Env, _Input) -> + mod_esi:deliver(Sid, ["OK"]). + start_apps(https) -> inets_test_lib:start_apps([crypto, public_key, ssl]); start_apps(sim_https) -> diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 05cf4f6cc3..dccdbfa94a 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.4.5 +INETS_VSN = 6.4.6 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 8477b0e148..1b72769ce3 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -981,8 +981,7 @@ f.txt: {person, "kalle", 25}. </item> <tag><c>eisdir</c></tag> <item> - <p>The named file is not a regular file. It can be a - directory, a FIFO, or a device.</p> + <p>The named file is a directory.</p> </item> <tag><c>enotdir</c></tag> <item> diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 418b0c50e1..f5a890cb95 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -340,8 +340,7 @@ handle_call(all_loaded, _From, S) -> {reply,all_loaded(Db),S}; handle_call({get_object_code,Mod}, _From, St) when is_atom(Mod) -> - Path = St#state.path, - case mod_to_bin(Path, Mod) of + case get_object_code(St, Mod) of {_,Bin,FName} -> {reply,{Mod,Bin,FName},St}; Error -> {reply,Error,St} end; @@ -1182,19 +1181,28 @@ load_file(Mod, From, St0) -> end, handle_pending_on_load(Action, Mod, From, St0). -load_file_1(Mod, From, #state{path=Path}=St) -> - case mod_to_bin(Path, Mod) of +load_file_1(Mod, From, St) -> + case get_object_code(St, Mod) of error -> {reply,{error,nofile},St}; {Mod,Binary,File} -> try_load_module_1(File, Mod, Binary, From, St) end. -mod_to_bin([Dir|Tail], Mod) -> - File = filename:append(Dir, atom_to_list(Mod) ++ objfile_extension()), +get_object_code(#state{path=Path}, Mod) when is_atom(Mod) -> + ModStr = atom_to_list(Mod), + case erl_prim_loader:is_basename(ModStr) of + true -> + mod_to_bin(Path, Mod, ModStr ++ objfile_extension()); + false -> + error + end. + +mod_to_bin([Dir|Tail], Mod, ModFile) -> + File = filename:append(Dir, ModFile), case erl_prim_loader:get_file(File) of error -> - mod_to_bin(Tail, Mod); + mod_to_bin(Tail, Mod, ModFile); {ok,Bin,_} -> case filename:pathtype(File) of absolute -> @@ -1203,10 +1211,9 @@ mod_to_bin([Dir|Tail], Mod) -> {Mod,Bin,absname(File)} end end; -mod_to_bin([], Mod) -> +mod_to_bin([], Mod, ModFile) -> %% At last, try also erl_prim_loader's own method - File = to_list(Mod) ++ objfile_extension(), - case erl_prim_loader:get_file(File) of + case erl_prim_loader:get_file(ModFile) of error -> error; % No more alternatives ! {ok,Bin,FName} -> diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 073b391288..f6791adf86 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -95,7 +95,11 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - +init_per_testcase(TC, Config) when TC == hostnames; + TC == nodenames -> + file:make_dir("hostnames_nodedir"), + file:write_file("hostnames_nodedir/ignore_core_files",""), + Config; init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Config. @@ -251,7 +255,7 @@ test_node(Name, Illigal) -> end, net_kernel:monitor_nodes(true), BinCommand = unicode:characters_to_binary(Command, utf8), - Prt = open_port({spawn, BinCommand}, [stream]), + Prt = open_port({spawn, BinCommand}, [stream,{cd,"hostnames_nodedir"}]), Node = list_to_atom(Name), receive {nodeup, Node} -> diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 3b502be8b8..ba0d075ef2 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -1083,11 +1083,9 @@ ifaddrs([{If,Opts}|IOs]) -> #ifopts{flags=F} = Ifopts = check_ifopts(Opts, #ifopts{name=If}), case F of {flags,Flags} -> - case lists:member(up, Flags) of - true -> - Ifopts#ifopts.addrs; - false -> - [] + case lists:member(running, Flags) of + true -> Ifopts#ifopts.addrs; + false -> [] end ++ ifaddrs(IOs); undefined -> ifaddrs(IOs) diff --git a/lib/observer/src/cdv_port_cb.erl b/lib/observer/src/cdv_port_cb.erl index b5cbe8132d..6bb8f07a74 100644 --- a/lib/observer/src/cdv_port_cb.erl +++ b/lib/observer/src/cdv_port_cb.erl @@ -34,7 +34,8 @@ -define(COL_CONN, ?COL_ID+1). -define(COL_NAME, ?COL_CONN+1). -define(COL_CTRL, ?COL_NAME+1). --define(COL_SLOT, ?COL_CTRL+1). +-define(COL_QUEUE, ?COL_CTRL+1). +-define(COL_SLOT, ?COL_QUEUE+1). @@ -44,6 +45,7 @@ col_to_elem(?COL_ID) -> #port.id; col_to_elem(?COL_CONN) -> #port.connected; col_to_elem(?COL_NAME) -> #port.name; col_to_elem(?COL_CTRL) -> #port.controls; +col_to_elem(?COL_QUEUE) -> #port.queue; col_to_elem(?COL_SLOT) -> #port.slot. col_spec() -> @@ -51,6 +53,7 @@ col_spec() -> {"Connected", ?wxLIST_FORMAT_LEFT, 120}, {"Name", ?wxLIST_FORMAT_LEFT, 150}, {"Controls", ?wxLIST_FORMAT_LEFT, 200}, + {"Queue", ?wxLIST_FORMAT_RIGHT, 100}, {"Slot", ?wxLIST_FORMAT_RIGHT, 50}]. get_info(_) -> @@ -96,9 +99,17 @@ format(D) -> info_fields() -> [{"Overview", [{"Name", name}, + {"State", state}, + {"Task Flags", task_flags}, {"Connected", {click,connected}}, {"Slot", slot}, - {"Controls", controls}]}, + {"Controls", controls}, + {"Input bytes", input}, + {"Output bytes", output}, + {"Queue bytes", queue}, + {"Port data", port_data}]}, {scroll_boxes, [{"Links",1,{click,links}}, - {"Monitors",1,{click,monitors}}]}]. + {"Monitors",1,{click,monitors}}, + {"Suspended",1,{click,suspended}} + ]}]. diff --git a/lib/observer/src/cdv_proc_cb.erl b/lib/observer/src/cdv_proc_cb.erl index f10650bbb7..0ea23dd7cb 100644 --- a/lib/observer/src/cdv_proc_cb.erl +++ b/lib/observer/src/cdv_proc_cb.erl @@ -149,6 +149,10 @@ info_fields() -> {"Old Heap", old_heap}, {"Heap Unused", heap_unused}, {"Old Heap Unused", old_heap_unused}, + {"Binary vheap", bin_vheap}, + {"Old Binary vheap", old_bin_vheap}, + {"Binary vheap unused", bin_vheap_unused}, + {"Old Binary vheap unused", old_bin_vheap_unused}, {"Number of Heap Fragements", num_heap_frag}, {"Heap Fragment Data",heap_frag_data}, {"New Heap Start", new_heap_start}, diff --git a/lib/observer/src/cdv_sched_cb.erl b/lib/observer/src/cdv_sched_cb.erl index 192aaf31a7..d2696a276f 100644 --- a/lib/observer/src/cdv_sched_cb.erl +++ b/lib/observer/src/cdv_sched_cb.erl @@ -31,7 +31,8 @@ %% Columns -define(COL_ID, 0). --define(COL_PROC, ?COL_ID+1). +-define(COL_TYPE, ?COL_ID+1). +-define(COL_PROC, ?COL_TYPE+1). -define(COL_PORT, ?COL_PROC+1). -define(COL_RQL, ?COL_PORT+1). -define(COL_PQL, ?COL_RQL+1). @@ -39,6 +40,7 @@ %% Callbacks for cdv_virtual_list_wx col_to_elem(id) -> col_to_elem(?COL_ID); col_to_elem(?COL_ID) -> #sched.name; +col_to_elem(?COL_TYPE) -> #sched.type; col_to_elem(?COL_PROC) -> #sched.process; col_to_elem(?COL_PORT) -> #sched.port; col_to_elem(?COL_RQL) -> #sched.run_q; @@ -46,6 +48,7 @@ col_to_elem(?COL_PQL) -> #sched.port_q. col_spec() -> [{"Id", ?wxLIST_FORMAT_RIGHT, 50}, + {"Type", ?wxLIST_FORMAT_CENTER, 100}, {"Current Process", ?wxLIST_FORMAT_CENTER, 130}, {"Current Port", ?wxLIST_FORMAT_CENTER, 130}, {"Run Queue Length", ?wxLIST_FORMAT_RIGHT, 180}, @@ -73,7 +76,8 @@ detail_pages() -> [{"Scheduler Information", fun init_gen_page/2}]. init_gen_page(Parent, Info0) -> - Fields = info_fields(), + Type = proplists:get_value(type, Info0), + Fields = info_fields(Type), Details = proplists:get_value(details, Info0), Info = if is_map(Details) -> Info0 ++ maps:to_list(Details); true -> Info0 @@ -81,15 +85,16 @@ init_gen_page(Parent, Info0) -> cdv_info_wx:start_link(Parent,{Fields,Info,[]}). %%% Internal -info_fields() -> +info_fields(Type) -> [{"Scheduler Overview", [{"Id", id}, + {"Type", type}, {"Current Process",process}, {"Current Port", port}, {"Sleep Info Flags", sleep_info}, {"Sleep Aux Work", sleep_aux} ]}, - {"Run Queues", + {run_queues_header(Type), [{"Flags", runq_flags}, {"Priority Max Length", runq_max}, {"Priority High Length", runq_high}, @@ -116,3 +121,8 @@ info_fields() -> {" ", {currp_stack, 11}} ]} ]. + +run_queues_header(normal) -> + "Run Queues"; +run_queues_header(DirtyX) -> + "Run Queues (common for all '" ++ atom_to_list(DirtyX) ++ "' schedulers)". diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index bba97624a7..d0c14db486 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -116,6 +116,10 @@ -define(allocator,allocator). -define(atoms,atoms). -define(binary,binary). +-define(dirty_cpu_scheduler,dirty_cpu_scheduler). +-define(dirty_cpu_run_queue,dirty_cpu_run_queue). +-define(dirty_io_scheduler,dirty_io_scheduler). +-define(dirty_io_run_queue,dirty_io_run_queue). -define(ende,ende). -define(erl_crash_dump,erl_crash_dump). -define(ets,ets). @@ -1222,6 +1226,18 @@ all_procinfo(Fd,Fun,Proc,WS,LineHead) -> "OldHeap unused" -> Bytes = list_to_integer(bytes(Fd))*WS, get_procinfo(Fd,Fun,Proc#proc{old_heap_unused=Bytes},WS); + "BinVHeap" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{bin_vheap=Bytes},WS); + "OldBinVHeap" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap=Bytes},WS); + "BinVHeap unused" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{bin_vheap_unused=Bytes},WS); + "OldBinVHeap unused" -> + Bytes = list_to_integer(bytes(Fd))*WS, + get_procinfo(Fd,Fun,Proc#proc{old_bin_vheap_unused=Bytes},WS); "New heap start" -> get_procinfo(Fd,Fun,Proc#proc{new_heap_start=bytes(Fd)},WS); "New heap top" -> @@ -1632,6 +1648,10 @@ port_to_tuple("#Port<"++Port) -> get_portinfo(Fd,Port) -> case line_head(Fd) of + "State" -> + get_portinfo(Fd,Port#port{state=bytes(Fd)}); + "Task Flags" -> + get_portinfo(Fd,Port#port{task_flags=bytes(Fd)}); "Slot" -> %% stored as integer so we can sort on it get_portinfo(Fd,Port#port{slot=list_to_integer(bytes(Fd))}); @@ -1656,6 +1676,10 @@ get_portinfo(Fd,Port) -> {Pid,Pid++" ("++Ref++")"} end || Mon <- Monitors0], get_portinfo(Fd,Port#port{monitors=Monitors}); + "Suspended" -> + Pids = split_pid_list_no_space(bytes(Fd)), + Suspended = [{Pid,Pid} || Pid <- Pids], + get_portinfo(Fd,Port#port{suspended=Suspended}); "Port controls linked-in driver" -> Str = lists:flatten(["Linked in driver: " | string(Fd)]), get_portinfo(Fd,Port#port{controls=Str}); @@ -1671,6 +1695,15 @@ get_portinfo(Fd,Port) -> "Port is UNIX fd not opened by emulator" -> Str = lists:flatten(["UNIX fd not opened by emulator: "| string(Fd)]), get_portinfo(Fd,Port#port{controls=Str}); + "Input" -> + get_portinfo(Fd,Port#port{input=list_to_integer(bytes(Fd))}); + "Output" -> + get_portinfo(Fd,Port#port{output=list_to_integer(bytes(Fd))}); + "Queue" -> + get_portinfo(Fd,Port#port{queue=list_to_integer(bytes(Fd))}); + "Port Data" -> + get_portinfo(Fd,Port#port{port_data=string(Fd)}); + "=" ++ _next_tag -> Port; Other -> @@ -2503,73 +2536,142 @@ get_indextableinfo1(Fd,IndexTable) -> %%----------------------------------------------------------------- %% Page with scheduler table information schedulers(File) -> - case lookup_index(?scheduler) of - [] -> - []; - Schedulers -> - Fd = open(File), - R = lists:map(fun({Name,Start}) -> - get_schedulerinfo(Fd,Name,Start) - end, - Schedulers), - close(Fd), - R - end. + Fd = open(File), -get_schedulerinfo(Fd,Name,Start) -> + Schds0 = case lookup_index(?scheduler) of + [] -> + []; + Normals -> + [{Normals, #sched{type=normal}}] + end, + Schds1 = case lookup_index(?dirty_cpu_scheduler) of + [] -> + Schds0; + DirtyCpus -> + [{DirtyCpus, get_dirty_runqueue(Fd, ?dirty_cpu_run_queue)} + | Schds0] + end, + Schds2 = case lookup_index(?dirty_io_scheduler) of + [] -> + Schds1; + DirtyIos -> + [{DirtyIos, get_dirty_runqueue(Fd, ?dirty_io_run_queue)} + | Schds1] + end, + + R = schedulers1(Fd, Schds2, []), + close(Fd), + R. + +schedulers1(_Fd, [], Acc) -> + Acc; +schedulers1(Fd, [{Scheds,Sched0} | Tail], Acc0) -> + Acc1 = lists:foldl(fun({Name,Start}, AccIn) -> + [get_schedulerinfo(Fd,Name,Start,Sched0) | AccIn] + end, + Acc0, + Scheds), + schedulers1(Fd, Tail, Acc1). + +get_schedulerinfo(Fd,Name,Start,Sched0) -> pos_bof(Fd,Start), - get_schedulerinfo1(Fd,#sched{name=Name}). + get_schedulerinfo1(Fd,Sched0#sched{name=list_to_integer(Name)}). -get_schedulerinfo1(Fd,Sched=#sched{details=Ds}) -> +sched_type(?dirty_cpu_run_queue) -> dirty_cpu; +sched_type(?dirty_io_run_queue) -> dirty_io. + +get_schedulerinfo1(Fd, Sched) -> + case get_schedulerinfo2(Fd, Sched) of + {more, Sched2} -> + get_schedulerinfo1(Fd, Sched2); + {done, Sched2} -> + Sched2 + end. + +get_schedulerinfo2(Fd, Sched=#sched{details=Ds}) -> case line_head(Fd) of "Current Process" -> - get_schedulerinfo1(Fd,Sched#sched{process=bytes(Fd, "None")}); + {more, Sched#sched{process=bytes(Fd, "None")}}; "Current Port" -> - get_schedulerinfo1(Fd,Sched#sched{port=bytes(Fd, "None")}); + {more, Sched#sched{port=bytes(Fd, "None")}}; + + "Scheduler Sleep Info Flags" -> + {more, Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}}; + "Scheduler Sleep Info Aux Work" -> + {more, Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}}; + + "Current Process State" -> + {more, Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}}; + "Current Process Internal State" -> + {more, Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}}; + "Current Process Program counter" -> + {more, Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}}; + "Current Process CP" -> + {more, Sched#sched{details=Ds#{currp_cp=>string(Fd)}}}; + "Current Process Limited Stack Trace" -> + %% If there shall be last in scheduler information block + {done, Sched#sched{details=get_limited_stack(Fd, 0, Ds)}}; + + "=" ++ _next_tag -> + {done, Sched}; + + Other -> + case Sched#sched.type of + normal -> + get_runqueue_info2(Fd, Other, Sched); + _ -> + unexpected(Fd,Other,"dirty scheduler information"), + {done, Sched} + end + end. + +get_dirty_runqueue(Fd, Tag) -> + case lookup_index(Tag) of + [{_, Start}] -> + pos_bof(Fd,Start), + get_runqueue_info1(Fd,#sched{type=sched_type(Tag)}); + [] -> + #sched{} + end. + +get_runqueue_info1(Fd, Sched) -> + case get_runqueue_info2(Fd, line_head(Fd), Sched) of + {more, Sched2} -> + get_runqueue_info1(Fd, Sched2); + {done, Sched2} -> + Sched2 + end. + +get_runqueue_info2(Fd, LineHead, Sched=#sched{details=Ds}) -> + case LineHead of "Run Queue Max Length" -> RQMax = list_to_integer(bytes(Fd)), RQ = RQMax + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}}; "Run Queue High Length" -> RQHigh = list_to_integer(bytes(Fd)), RQ = RQHigh + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}}; "Run Queue Normal Length" -> RQNorm = list_to_integer(bytes(Fd)), RQ = RQNorm + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}}; "Run Queue Low Length" -> RQLow = list_to_integer(bytes(Fd)), RQ = RQLow + Sched#sched.run_q, - get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}); + {more, Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}}; "Run Queue Port Length" -> RQ = list_to_integer(bytes(Fd)), - get_schedulerinfo1(Fd,Sched#sched{port_q=RQ}); - - "Scheduler Sleep Info Flags" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_info=>bytes(Fd, "None")}}); - "Scheduler Sleep Info Aux Work" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_aux=>bytes(Fd, "None")}}); + {more, Sched#sched{port_q=RQ}}; "Run Queue Flags" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}); + {more, Sched#sched{details=Ds#{runq_flags=>bytes(Fd, "None")}}}; - "Current Process State" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_state=>bytes(Fd)}}); - "Current Process Internal State" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_int_state=>bytes(Fd)}}); - "Current Process Program counter" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_prg_cnt=>string(Fd)}}); - "Current Process CP" -> - get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_cp=>string(Fd)}}); - "Current Process Limited Stack Trace" -> - %% If there shall be last in scheduler information block - Sched#sched{details=get_limited_stack(Fd, 0, Ds)}; "=" ++ _next_tag -> - Sched; + {done, Sched}; Other -> unexpected(Fd,Other,"scheduler information"), - Sched + {done, Sched} end. get_limited_stack(Fd, N, Ds) -> @@ -3000,6 +3102,10 @@ tag_to_atom("allocated_areas") -> ?allocated_areas; tag_to_atom("allocator") -> ?allocator; tag_to_atom("atoms") -> ?atoms; tag_to_atom("binary") -> ?binary; +tag_to_atom("dirty_cpu_scheduler") -> ?dirty_cpu_scheduler; +tag_to_atom("dirty_cpu_run_queue") -> ?dirty_cpu_run_queue; +tag_to_atom("dirty_io_scheduler") -> ?dirty_io_scheduler; +tag_to_atom("dirty_io_run_queue") -> ?dirty_io_run_queue; tag_to_atom("end") -> ?ende; tag_to_atom("erl_crash_dump") -> ?erl_crash_dump; tag_to_atom("ets") -> ?ets; diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl index 6a93a089fd..252e19379d 100644 --- a/lib/observer/src/crashdump_viewer.hrl +++ b/lib/observer/src/crashdump_viewer.hrl @@ -80,6 +80,10 @@ old_heap, heap_unused, old_heap_unused, + bin_vheap, + old_bin_vheap, + bin_vheap_unused, + old_bin_vheap_unused, new_heap_start, new_heap_top, stack_top, @@ -95,19 +99,27 @@ -record(port, {id, + state, + task_flags=0, slot, connected, links, name, monitors, - controls}). + suspended, + controls, + input, + output, + queue, + port_data}). -record(sched, {name, + type, process, port, run_q=0, - port_q=0, + port_q, details=#{} }). diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl index 119d7cc3d4..136ee55b54 100644 --- a/lib/runtime_tools/src/system_information.erl +++ b/lib/runtime_tools/src/system_information.erl @@ -488,10 +488,10 @@ to_fd(Fd) -> ok = file:write(Fd, io_lib:format(Format, Args)) end, - EmitChunk("{system_information_version, ~p}.~n" + EmitChunk("{system_information_version, ~w}.~n" "{system_information,[" - "{init_arguments,~p}," - "{code_paths,~p},", + "{init_arguments,~w}," + "{code_paths,~w},", [?REPORT_FILE_VSN, init:get_arguments(), code:get_path()]), @@ -499,12 +499,12 @@ to_fd(Fd) -> emit_code_info(EmitChunk), EmitChunk( "," %% Note the leading comma! - "{system_info,~p}," - "{erts_compile_info,~p}," - "{beam_dynamic_libraries,~p}," - "{environment_erts,~p}," - "{environment,~p}," - "{sanity_check,~p}" + "{system_info,~w}," + "{erts_compile_info,~w}," + "{beam_dynamic_libraries,~w}," + "{environment_erts,~w}," + "{environment,~w}," + "{sanity_check,~w}" "]}.~n", [erlang_system_info(), erlang:system_info(compile_info), @@ -533,11 +533,11 @@ emit_application_info(EmitChunk, Path) -> Description = proplists:get_value(description, Info, []), Version = proplists:get_value(vsn, Info, []), - EmitChunk("{application, {~p,[" - "{description,~p}," - "{vsn,~p}," - "{path,~p}," - "{runtime_dependencies,~p},", + EmitChunk("{application, {~w,[" + "{description,~w}," + "{vsn,~w}," + "{path,~w}," + "{runtime_dependencies,~w},", [App, Description, Version, Path, RtDeps]), emit_module_info_from_path(EmitChunk, Path), EmitChunk("]}}", []) @@ -545,7 +545,7 @@ emit_application_info(EmitChunk, Path) -> emit_code_path_info(EmitChunk, Path) -> EmitChunk("{code, [" - "{path, ~p},", [Path]), + "{path, ~w},", [Path]), emit_module_info_from_path(EmitChunk, Path), EmitChunk("]}", []). @@ -573,11 +573,11 @@ emit_module_info(EmitChunk, Beam) -> _ -> true end, - EmitChunk("{~p,[" - "{loaded,~p}," - "{native,~p}," - "{compiler,~p}," - "{md5,~p}" + EmitChunk("{~w,[" + "{loaded,~w}," + "{native,~w}," + "{compiler,~w}," + "{md5,~w}" "]}", [Mod, Loaded, Native, CompilerVersion, hexstring(Md5)]). diff --git a/lib/snmp/doc/src/snmp_impl_example_agent.xml b/lib/snmp/doc/src/snmp_impl_example_agent.xml index a86006a0a7..e576fa51f3 100644 --- a/lib/snmp/doc/src/snmp_impl_example_agent.xml +++ b/lib/snmp/doc/src/snmp_impl_example_agent.xml @@ -47,6 +47,7 @@ EX1-MIB DEFINITIONS ::= BEGIN IMPORTS + experimental FROM RFC1155-SMI RowStatus FROM STANDARD-MIB DisplayString FROM RFC1213-MIB OBJECT-TYPE FROM RFC-1212 @@ -81,7 +82,7 @@ EX1-MIB DEFINITIONS ::= BEGIN FriendsEntry ::= SEQUENCE { - fIndex + fIndex INTEGER, fName DisplayString, @@ -105,6 +106,7 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "Name of friend" ::= { friendsEntry 2 } + fAddress OBJECT-TYPE SYNTAX DisplayString (SIZE (0..255)) ACCESS read-write @@ -112,6 +114,7 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "Address of friend" ::= { friendsEntry 3 } + fStatus OBJECT-TYPE SYNTAX RowStatus ACCESS read-write @@ -119,12 +122,13 @@ EX1-MIB DEFINITIONS ::= BEGIN DESCRIPTION "The status of this conceptual row." ::= { friendsEntry 4 } + fTrap TRAP-TYPE ENTERPRISE example1 VARIABLES { myName, fIndex } DESCRIPTION - "This trap is sent when something happens to - the friend specified by fIndex." + "This trap is sent when something happens to + the friend specified by fIndex." ::= 1 END </code> diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 3a2f55a487..c559da911b 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,42 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.6.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Adjusted supervisor timeouts</p> + <p> + Own Id: OTP-14907</p> + </item> + <item> + <p> + Remove ERROR messages for slow process exits</p> + <p> + Own Id: OTP-14930</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add option <c>save_accepted_host</c> to + <c>ssh:connection</c>. This option, if set to false, + inhibits saving host keys to e.g the file + <c>known_hosts</c>.</p> + <p> + Own Id: OTP-14935</p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.6.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 337f4094cc..acf94ff6af 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -227,6 +227,18 @@ </item> </list> </item> + + <tag><c><![CDATA[{save_accepted_host, boolean()}]]></c></tag> + <item> + <p>If <c>true</c>, the client saves an accepted host key to avoid the + accept question the next time the same host is connected. If the option + <c>key_cb</c> is not present, the key is saved in the file "known_hosts". + </p> + <p>If <c>false</c>, the key is not saved and the key will still be unknown + at the next access of the same host. + </p> + </item> + <tag><c><![CDATA[{user_interaction, boolean()}]]></c></tag> <item> <p>If <c>false</c>, disables the client to connect to the server diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index ed7fbf9cf3..129426a6d5 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -464,11 +464,16 @@ <v>FileInfo = record()</v> </type> <desc> - <p>Returns a <c><![CDATA[file_info]]></c> record from the file specified by + <p>Returns a <c><![CDATA[file_info]]></c> record from the file system object specified by <c><![CDATA[Name]]></c> or <c><![CDATA[Handle]]></c>. See <seealso marker="kernel:file#read_file_info-2">file:read_file_info/2</seealso> for information about the record. </p> + <p> + Depending on the underlying OS:es links might be followed and info on the final file, directory + etc is returned. See <seealso marker="#read_link_info-2">ssh_sftp::read_link_info/2</seealso> + on how to get information on links instead. + </p> </desc> </func> diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index a24664793b..fc564a359b 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -86,10 +86,7 @@ child_spec(Address, Port, Profile, Options) -> Timeout = ?GET_INTERNAL_OPT(timeout, Options, ?DEFAULT_TIMEOUT), #{id => id(Address, Port, Profile), start => {ssh_acceptor, start_link, [Port, Address, Options, Timeout]}, - restart => transient, - shutdown => 5500, %brutal_kill, - type => worker, - modules => [ssh_acceptor] + restart => transient % because a crashed listener could be replaced by a new one }. id(Address, Port, Profile) -> diff --git a/lib/ssh/src/ssh_channel_sup.erl b/lib/ssh/src/ssh_channel_sup.erl index 6b01dc334d..8444533fd1 100644 --- a/lib/ssh/src/ssh_channel_sup.erl +++ b/lib/ssh/src/ssh_channel_sup.erl @@ -26,7 +26,7 @@ -behaviour(supervisor). --export([start_link/1, start_child/2]). +-export([start_link/1, start_child/5]). %% Supervisor callback -export([init/1]). @@ -37,7 +37,14 @@ start_link(Args) -> supervisor:start_link(?MODULE, [Args]). -start_child(Sup, ChildSpec) -> +start_child(Sup, Callback, Id, Args, Exec) -> + ChildSpec = + #{id => make_ref(), + start => {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, + restart => temporary, + type => worker, + modules => [ssh_channel] + }, supervisor:start_child(Sup, ChildSpec). %%%========================================================================= diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 7e9ee78fd2..946ae2967b 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -812,22 +812,20 @@ start_channel(Cb, Id, Args, SubSysSup, Opts) -> start_channel(Cb, Id, Args, SubSysSup, undefined, Opts). start_channel(Cb, Id, Args, SubSysSup, Exec, Opts) -> - ChildSpec = child_spec(Cb, Id, Args, Exec), ChannelSup = ssh_subsystem_sup:channel_supervisor(SubSysSup), - assert_limit_num_channels_not_exceeded(ChannelSup, Opts), - ssh_channel_sup:start_child(ChannelSup, ChildSpec). + case max_num_channels_not_exceeded(ChannelSup, Opts) of + true -> + ssh_channel_sup:start_child(ChannelSup, Cb, Id, Args, Exec); + false -> + throw(max_num_channels_exceeded) + end. -assert_limit_num_channels_not_exceeded(ChannelSup, Opts) -> +max_num_channels_not_exceeded(ChannelSup, Opts) -> MaxNumChannels = ?GET_OPT(max_channels, Opts), NumChannels = length([x || {_,_,worker,[ssh_channel]} <- supervisor:which_children(ChannelSup)]), - if - %% Note that NumChannels is BEFORE starting a new one - NumChannels < MaxNumChannels -> - ok; - true -> - throw(max_num_channels_exceeded) - end. + %% Note that NumChannels is BEFORE starting a new one + NumChannels < MaxNumChannels. %%-------------------------------------------------------------------- %%% Internal functions @@ -874,14 +872,6 @@ check_subsystem(SsName, Options) -> Value end. -child_spec(Callback, Id, Args, Exec) -> - Name = make_ref(), - StartFunc = {ssh_channel, start_link, [self(), Id, Callback, Args, Exec]}, - Restart = temporary, - Shutdown = 3600, - Type = worker, - {Name, StartFunc, Restart, Shutdown, Type, [ssh_channel]}. - start_cli(#connection{cli_spec = no_cli}, _) -> {error, cli_disabled}; start_cli(#connection{options = Options, diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 0ca960ef96..c8ac3a9c04 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1460,13 +1460,12 @@ terminate(shutdown, StateName, State0) -> State0), finalize_termination(StateName, State); -%% terminate({shutdown,Msg}, StateName, State0) when is_record(Msg,ssh_msg_disconnect)-> -%% State = send_msg(Msg, State0), -%% finalize_termination(StateName, Msg, State); - terminate({shutdown,_R}, StateName, State) -> finalize_termination(StateName, State); +terminate(kill, StateName, State) -> + finalize_termination(StateName, State); + terminate(Reason, StateName, State0) -> %% Others, e.g undef, {badmatch,_} log_error(Reason), diff --git a/lib/ssh/src/ssh_connection_sup.erl b/lib/ssh/src/ssh_connection_sup.erl index 60ee8b7c73..2e8450090a 100644 --- a/lib/ssh/src/ssh_connection_sup.erl +++ b/lib/ssh/src/ssh_connection_sup.erl @@ -52,10 +52,7 @@ init(_) -> }, ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. start => {ssh_connection_handler, start_link, []}, - restart => temporary, - shutdown => 4000, - type => worker, - modules => [ssh_connection_handler] + restart => temporary % because there is no way to restart a crashed connection } ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index 68c99743ee..cf1534bd78 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -439,6 +439,12 @@ default(client) -> class => user_options }, + {save_accepted_host, def} => + #{default => true, + chk => fun erlang:is_boolean/1, + class => user_options + }, + {pref_public_key_algs, def} => #{default => ssh_transport:default_algorithms(public_key), chk => fun check_pref_public_key_algs/1, diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index 8db051095c..77da240a66 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -74,18 +74,14 @@ ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_connection_sup, Address, Port), start => {ssh_connection_sup, start_link, [Options]}, restart => temporary, - shutdown => 5000, - type => supervisor, - modules => [ssh_connection_sup] + type => supervisor }. ssh_channel_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_channel_sup, Address, Port), start => {ssh_channel_sup, start_link, [Options]}, restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_channel_sup] + type => supervisor }. id(Role, Sup, Address, Port) -> diff --git a/lib/ssh/src/ssh_sup.erl b/lib/ssh/src/ssh_sup.erl index eaec7a54e4..8183016ba5 100644 --- a/lib/ssh/src/ssh_sup.erl +++ b/lib/ssh/src/ssh_sup.erl @@ -36,15 +36,14 @@ init(_) -> intensity => 10, period => 3600 }, - ChildSpecs = [#{id => Module, - start => {Module, start_link, []}, - restart => permanent, - shutdown => 4000, %brutal_kill, - type => supervisor, - modules => [Module] + ChildSpecs = [#{id => sshd_sup, + start => {sshd_sup, start_link, []}, + type => supervisor + }, + #{id => sshc_sup, + start => {sshc_sup, start_link, []}, + type => supervisor } - || Module <- [sshd_sup, - sshc_sup] ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index e70abf59c2..17f990c5d8 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -63,9 +63,7 @@ init([Address, Port, Profile, Options]) -> [#{id => id(ssh_acceptor_sup, Address, Port, Profile), start => {ssh_acceptor_sup, start_link, [Address, Port, Profile, Options]}, restart => transient, - shutdown => infinity, - type => supervisor, - modules => [ssh_acceptor_sup] + type => supervisor }]; _ -> [] @@ -124,9 +122,8 @@ start_subsystem(SystemSup, Role, Address, Port, Profile, Options) -> #{id => make_ref(), start => {ssh_subsystem_sup, start_link, [Role, Address, Port, Profile, Options]}, restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_subsystem_sup]}, + type => supervisor + }, supervisor:start_child(SystemSup, SubsystemSpec). stop_subsystem(SystemSup, SubSys) -> diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index ad9efc4755..975053d301 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -889,10 +889,13 @@ known_host_key(#ssh{opts = Opts, key_cb = {KeyCb,KeyCbOpts}, peer = {PeerName,_} {_,true} -> ok; {_,false} -> + DoAdd = ?GET_OPT(save_accepted_host, Opts), case accepted_host(Ssh, PeerName, Public, Opts) of - true -> + true when DoAdd == true -> {_,R} = add_host_key(KeyCb, PeerName, Public, [{key_cb_private,KeyCbOpts}|UserOpts]), R; + true when DoAdd == false -> + ok; false -> {error, rejected_by_user}; {error,E} -> diff --git a/lib/ssh/src/sshc_sup.erl b/lib/ssh/src/sshc_sup.erl index 133b2c6450..fd4d8a3c07 100644 --- a/lib/ssh/src/sshc_sup.erl +++ b/lib/ssh/src/sshc_sup.erl @@ -60,10 +60,7 @@ init(_) -> }, ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. start => {ssh_connection_handler, start_link, []}, - restart => temporary, - shutdown => 4000, - type => worker, - modules => [ssh_connection_handler] + restart => temporary % because there is no way to restart a crashed connection } ], {ok, {SupFlags,ChildSpecs}}. diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index c23e65d955..779a861a54 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -90,10 +90,8 @@ init(_) -> child_spec(Address, Port, Profile, Options) -> #{id => id(Address, Port, Profile), start => {ssh_system_sup, start_link, [Address, Port, Profile, Options]}, - restart => temporary, - shutdown => infinity, - type => supervisor, - modules => [ssh_system_sup] + restart => temporary, + type => supervisor }. id(Address, Port, Profile) -> diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index 202b0afe57..d16b169da3 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -764,11 +764,11 @@ cli(Config) when is_list(Config) -> {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), ssh_connection:shell(ConnectionRef, ChannelId), - ok = ssh_connection:send(ConnectionRef, ChannelId, <<"q">>), + ssh_connection:send(ConnectionRef, ChannelId, <<"q">>), receive {ssh_cm, ConnectionRef, {data,0,0, <<"\r\nYou are accessing a dummy, type \"q\" to exit\r\n\n">>}} -> - ok = ssh_connection:send(ConnectionRef, ChannelId, <<"q">>) + ssh_connection:send(ConnectionRef, ChannelId, <<"q">>) after 30000 -> ct:fail("timeout ~p:~p",[?MODULE,?LINE]) end, @@ -1491,7 +1491,7 @@ new_do_shell(IO, N, Ops=[{Order,Arg}|More]) -> ct:fail("*** Expected ~p, but got ~p",[string:strip(ExpStr),RecStr]) end after 30000 -> - ct:log("Meassage queue of ~p:~n~p", + ct:log("Message queue of ~p:~n~p", [self(), erlang:process_info(self(), messages)]), case Order of expect -> ct:fail("timeout, expected ~p",[string:strip(Arg)]); diff --git a/lib/ssh/test/ssh_compat_SUITE.erl b/lib/ssh/test/ssh_compat_SUITE.erl index 82b83dd83d..f7eda1dc08 100644 --- a/lib/ssh/test/ssh_compat_SUITE.erl +++ b/lib/ssh/test/ssh_compat_SUITE.erl @@ -92,6 +92,7 @@ end_per_suite(Config) -> %%% os:cmd("docker rm $(docker ps -aq -f status=exited)"), %% Remove dangling images: %%% os:cmd("docker rmi $(docker images -f dangling=true -q)"), + catch ssh:stop(), Config. diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index ba4518cfe6..77c299b3ee 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -85,6 +85,7 @@ init_per_suite(Config) -> ?CHECK_CRYPTO(Config). end_per_suite(Config) -> + catch ssh:stop(), Config. %%-------------------------------------------------------------------- @@ -800,6 +801,8 @@ stop_listener(Config) when is_list(Config) -> ssh:stop_daemon(Pid0), ssh:stop_daemon(Pid1); Error -> + ssh:close(ConnectionRef0), + ssh:stop_daemon(Pid0), ct:fail({unexpected, Error}) end. diff --git a/lib/ssh/test/ssh_engine_SUITE.erl b/lib/ssh/test/ssh_engine_SUITE.erl index daf93891e9..c131a70973 100644 --- a/lib/ssh/test/ssh_engine_SUITE.erl +++ b/lib/ssh/test/ssh_engine_SUITE.erl @@ -55,16 +55,22 @@ basic_tests() -> init_per_suite(Config) -> ssh:start(), ?CHECK_CRYPTO( - case load_engine() of - {ok,E} -> - [{engine,E}|Config]; - {error, notsup} -> - {skip, "Engine not supported on this OpenSSL version"}; - {error, bad_engine_id} -> - {skip, "Dynamic Engine not supported"}; - Other -> - ct:log("Engine load failed: ~p",[Other]), - {fail, "Engine load failed"} + case crypto:info_lib() of + [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + {skip, "Strange Engine stuff"}; + + _ -> + case load_engine() of + {ok,E} -> + [{engine,E}|Config]; + {error, notsup} -> + {skip, "Engine not supported on this OpenSSL version"}; + {error, bad_engine_id} -> + {skip, "Dynamic Engine not supported"}; + Other -> + ct:log("Engine load failed: ~p",[Other]), + {fail, "Engine load failed"} + end end ). diff --git a/lib/ssh/test/ssh_options_SUITE.erl b/lib/ssh/test/ssh_options_SUITE.erl index 144ec7f8fd..12a85c40aa 100644 --- a/lib/ssh/test/ssh_options_SUITE.erl +++ b/lib/ssh/test/ssh_options_SUITE.erl @@ -70,7 +70,8 @@ hostkey_fingerprint_check_sha256/1, hostkey_fingerprint_check_sha384/1, hostkey_fingerprint_check_sha512/1, - hostkey_fingerprint_check_list/1 + hostkey_fingerprint_check_list/1, + save_accepted_host_option/1 ]). %%% Common test callbacks @@ -124,6 +125,7 @@ all() -> id_string_own_string_server, id_string_own_string_server_trail_space, id_string_random_server, + save_accepted_host_option, {group, hardening_tests} ]. @@ -206,32 +208,23 @@ end_per_group(_, Config) -> %%-------------------------------------------------------------------- init_per_testcase(_TestCase, Config) -> ssh:start(), - Config. - -end_per_testcase(TestCase, Config) when TestCase == server_password_option; - TestCase == server_userpassword_option; - TestCase == server_pwdfun_option; - TestCase == server_pwdfun_4_option -> + %% Create a clean user_dir UserDir = filename:join(proplists:get_value(priv_dir, Config), nopubkey), ssh_test_lib:del_dirs(UserDir), - end_per_testcase(Config); -end_per_testcase(_TestCase, Config) -> - end_per_testcase(Config). + file:make_dir(UserDir), + [{user_dir,UserDir}|Config]. -end_per_testcase(_Config) -> +end_per_testcase(_TestCase, Config) -> ssh:stop(), ok. %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- %%% validate to server that uses the 'password' option server_password_option(Config) when is_list(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, {user_dir, UserDir}, @@ -262,12 +255,10 @@ server_password_option(Config) when is_list(Config) -> %%% validate to server that uses the 'password' option server_userpassword_option(Config) when is_list(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {user_passwords, [{"vego", "morot"}]}]), ConnectionRef = @@ -297,15 +288,13 @@ server_userpassword_option(Config) when is_list(Config) -> %%-------------------------------------------------------------------- %%% validate to server that uses the 'pwdfun' option server_pwdfun_option(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), CHKPWD = fun("foo",Pwd) -> Pwd=="bar"; (_,_) -> false end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {pwdfun,CHKPWD}]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, @@ -335,9 +324,7 @@ server_pwdfun_option(Config) -> %%-------------------------------------------------------------------- %%% validate to server that uses the 'pwdfun/4' option server_pwdfun_4_option(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), PWDFUN = fun("foo",Pwd,{_,_},undefined) -> Pwd=="bar"; ("fie",Pwd,{_,_},undefined) -> {Pwd=="bar",new_state}; @@ -345,7 +332,7 @@ server_pwdfun_4_option(Config) -> (_,_,_,_) -> false end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {pwdfun,PWDFUN}]), ConnectionRef1 = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, @@ -395,9 +382,7 @@ server_pwdfun_4_option(Config) -> %%-------------------------------------------------------------------- server_pwdfun_4_option_repeat(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), %% Test that the state works Parent = self(), @@ -406,7 +391,7 @@ server_pwdfun_4_option_repeat(Config) -> (_,P,_,S) -> Parent!{P,S}, {false,S+1} end, {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, PrivDir}, + {user_dir, UserDir}, {auth_methods,"keyboard-interactive"}, {pwdfun,PWDFUN}]), @@ -490,9 +475,7 @@ user_dir_option(Config) -> %%-------------------------------------------------------------------- %%% validate client that uses the 'ssh_msg_debug_fun' option ssh_msg_debug_fun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, @@ -530,9 +513,7 @@ ssh_msg_debug_fun_option_client(Config) -> %%-------------------------------------------------------------------- connectfun_disconnectfun_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -576,9 +557,7 @@ connectfun_disconnectfun_server(Config) -> %%-------------------------------------------------------------------- connectfun_disconnectfun_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -607,9 +586,7 @@ connectfun_disconnectfun_client(Config) -> %%-------------------------------------------------------------------- %%% validate client that uses the 'ssh_msg_debug_fun' option ssh_msg_debug_fun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -651,9 +628,7 @@ ssh_msg_debug_fun_option_server(Config) -> %%-------------------------------------------------------------------- disconnectfun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -686,9 +661,7 @@ disconnectfun_option_server(Config) -> %%-------------------------------------------------------------------- disconnectfun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -720,9 +693,7 @@ disconnectfun_option_client(Config) -> %%-------------------------------------------------------------------- unexpectedfun_option_server(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -763,9 +734,7 @@ unexpectedfun_option_server(Config) -> %%-------------------------------------------------------------------- unexpectedfun_option_client(Config) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), Parent = self(), @@ -840,14 +809,9 @@ supported_hash(HashAlg) -> really_do_hostkey_fingerprint_check(Config, HashAlg) -> - PrivDir = proplists:get_value(priv_dir, Config), - UserDirServer = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDirServer), + UserDir = proplists:get_value(user_dir, Config), SysDir = proplists:get_value(data_dir, Config), - UserDirClient = - ssh_test_lib:create_random_dir(Config), % Ensure no 'known_hosts' disturbs - %% All host key fingerprints. Trust that public_key has checked the ssh_hostkey_fingerprint %% function since that function is used by the ssh client... FPs0 = [case HashAlg of @@ -873,7 +837,7 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> %% Start daemon with the public keys that we got fingerprints from {Pid, Host0, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, - {user_dir, UserDirServer}, + {user_dir, UserDir}, {password, "morot"}]), Host = ssh_test_lib:ntoa(Host0), FP_check_fun = fun(PeerName, FP) -> @@ -896,7 +860,8 @@ really_do_hostkey_fingerprint_check(Config, HashAlg) -> end}, {user, "foo"}, {password, "morot"}, - {user_dir, UserDirClient}, + {user_dir, UserDir}, + {save_accepted_host, false}, % Ensure no 'known_hosts' disturbs {user_interaction, false}]), ssh:stop_daemon(Pid). @@ -987,9 +952,7 @@ ms_passed(T0) -> %%-------------------------------------------------------------------- ssh_daemon_minimal_remote_max_packet_size_option(Config) -> SystemDir = proplists:get_value(data_dir, Config), - PrivDir = proplists:get_value(priv_dir, Config), - UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth - file:make_dir(UserDir), + UserDir = proplists:get_value(user_dir, Config), {Server, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir}, @@ -1314,6 +1277,33 @@ try_to_connect(Connect, Host, Port, Pid, Tref, N) -> end. %%-------------------------------------------------------------------- +save_accepted_host_option(Config) -> + UserDir = proplists:get_value(user_dir, Config), + KnownHosts = filename:join(UserDir, "known_hosts"), + SysDir = proplists:get_value(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]} + ]), + {error,enoent} = file:read_file(KnownHosts), + + {ok,_C1} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}, + {save_accepted_host, false}, + {user_dir, UserDir}]), + {error,enoent} = file:read_file(KnownHosts), + + {ok,_C2} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "vego"}, + {password, "morot"}, + {user_interaction, false}, + {user_dir, UserDir}]), + {ok,_} = file:read_file(KnownHosts), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index 3920a1c592..1df55834b1 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -42,7 +42,9 @@ suite() -> all() -> [default_tree, sshc_subtree, sshd_subtree, sshd_subtree_profile, - killed_acceptor_restarts]. + killed_acceptor_restarts, + shell_channel_tree + ]. groups() -> []. @@ -199,7 +201,7 @@ killed_acceptor_restarts(Config) -> Port2 = ssh_test_lib:daemon_port(DaemonPid2), true = (Port /= Port2), - ct:pal("~s",[lists:flatten(ssh_info:string())]), + ct:log("~s",[lists:flatten(ssh_info:string())]), {ok,[{AccPid,ListenAddr,Port}]} = acceptor_pid(DaemonPid), {ok,[{AccPid2,ListenAddr,Port2}]} = acceptor_pid(DaemonPid2), @@ -216,11 +218,14 @@ killed_acceptor_restarts(Config) -> %% Make acceptor restart: exit(AccPid, kill), + ?wait_match(undefined, process_info(AccPid)), %% Check it is a new acceptor: - {ok,[{AccPid1,ListenAddr,Port}]} = acceptor_pid(DaemonPid), - true = (AccPid /= AccPid1), - true = (AccPid2 /= AccPid1), + ?wait_match({ok,[{AccPid1,ListenAddr,Port}]}, AccPid1=/=AccPid, + acceptor_pid(DaemonPid), + AccPid1, + 500, 30), + AccPid1 =/= AccPid2, %% Connect second client and check it is alive: {ok,C2} = ssh:connect("localhost", Port, [{silently_accept_hosts, true}, @@ -230,21 +235,113 @@ killed_acceptor_restarts(Config) -> {user_dir, UserDir}]), [{client_version,_}] = ssh:connection_info(C2,[client_version]), - ct:pal("~s",[lists:flatten(ssh_info:string())]), + ct:log("~s",[lists:flatten(ssh_info:string())]), %% Check first client is still alive: [{client_version,_}] = ssh:connection_info(C1,[client_version]), ok = ssh:stop_daemon(DaemonPid2), - timer:sleep(15000), + ?wait_match(undefined, process_info(DaemonPid2), 1000, 30), [{client_version,_}] = ssh:connection_info(C1,[client_version]), [{client_version,_}] = ssh:connection_info(C2,[client_version]), ok = ssh:stop_daemon(DaemonPid), - timer:sleep(15000), + ?wait_match(undefined, process_info(DaemonPid), 1000, 30), {error,closed} = ssh:connection_info(C1,[client_version]), {error,closed} = ssh:connection_info(C2,[client_version]). + +%%------------------------------------------------------------------------- +shell_channel_tree(Config) -> + PrivDir = proplists:get_value(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = proplists:get_value(data_dir, Config), + TimeoutShell = + fun() -> + io:format("TimeoutShell started!~n",[]), + timer:sleep(5000), + ct:log("~p TIMEOUT!",[self()]) + end, + {Daemon, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {shell, fun(_User) -> + spawn(TimeoutShell) + end + } + ]), + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_interaction, true}, + {user_dir, UserDir}]), + + [ChannelSup|_] = Sups0 = chk_empty_con_daemon(Daemon), + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + ok = ssh_connection:shell(ConnectionRef,ChannelId0), + + ?wait_match([{_, GroupPid,worker,[ssh_channel]}], + supervisor:which_children(ChannelSup), + [GroupPid]), + {links,GroupLinks} = erlang:process_info(GroupPid, links), + [ShellPid] = GroupLinks--[ChannelSup], + ct:log("GroupPid = ~p, ShellPid = ~p",[GroupPid,ShellPid]), + + receive + {ssh_cm,ConnectionRef, {data, ChannelId0, 0, <<"TimeoutShell started!\r\n">>}} -> + receive + %%---- wait for the subsystem to terminate + {ssh_cm,ConnectionRef,{closed,ChannelId0}} -> + ct:log("Subsystem terminated",[]), + case {chk_empty_con_daemon(Daemon), + process_info(GroupPid), + process_info(ShellPid)} of + {Sups0, undefined, undefined} -> + %% SUCCESS + ssh:stop_daemon(Daemon); + {Sups0, _, undefined} -> + ssh:stop_daemon(Daemon), + ct:fail("Group proc lives!"); + {Sups0, undefined, _} -> + ssh:stop_daemon(Daemon), + ct:fail("Shell proc lives!"); + _ -> + ssh:stop_daemon(Daemon), + ct:fail("Sup tree changed!") + end + after 10000 -> + ssh:close(ConnectionRef), + ssh:stop_daemon(Daemon), + ct:fail("CLI Timeout") + end + after 10000 -> + ssh:close(ConnectionRef), + ssh:stop_daemon(Daemon), + ct:fail("CLI Timeout") + end. + + +chk_empty_con_daemon(Daemon) -> + ?wait_match([{_,SubSysSup, supervisor,[ssh_subsystem_sup]}, + {{ssh_acceptor_sup,_,_,_}, AccSup, supervisor,[ssh_acceptor_sup]}], + supervisor:which_children(Daemon), + [SubSysSup,AccSup]), + ?wait_match([{{server,ssh_connection_sup, _,_}, + ConnectionSup, supervisor, + [ssh_connection_sup]}, + {{server,ssh_channel_sup,_ ,_}, + ChannelSup,supervisor, + [ssh_channel_sup]}], + supervisor:which_children(SubSysSup), + [ConnectionSup,ChannelSup]), + ?wait_match([{{ssh_acceptor_sup,_,_,_},_,worker,[ssh_acceptor]}], + supervisor:which_children(AccSup)), + ?wait_match([{_, _, worker,[ssh_connection_handler]}], + supervisor:which_children(ConnectionSup)), + ?wait_match([], supervisor:which_children(ChannelSup)), + [ChannelSup, ConnectionSup, SubSysSup, AccSup]. + %%------------------------------------------------------------------------- %% Help functions %%------------------------------------------------------------------------- diff --git a/lib/ssh/test/ssh_test_lib.hrl b/lib/ssh/test/ssh_test_lib.hrl index eaf856e6e8..4b6579bd71 100644 --- a/lib/ssh/test/ssh_test_lib.hrl +++ b/lib/ssh/test/ssh_test_lib.hrl @@ -16,12 +16,12 @@ %%------------------------------------------------------------------------- %% Help macro %%------------------------------------------------------------------------- --define(wait_match(Pattern, FunctionCall, Bind, Timeout, Ntries), +-define(wait_match(Pattern, Guard, FunctionCall, Bind, Timeout, Ntries), Bind = (fun() -> F = fun(N, F1) -> case FunctionCall of - Pattern -> Bind; + Pattern when Guard -> Bind; _ when N>0 -> ct:pal("Must sleep ~p ms at ~p:~p",[Timeout,?MODULE,?LINE]), timer:sleep(Timeout), @@ -34,6 +34,9 @@ end)() ). +-define(wait_match(Pattern, FunctionCall, Bind, Timeout, Ntries), + ?wait_match(Pattern, true, FunctionCall, Bind, Timeout, Ntries)). + -define(wait_match(Pattern, FunctionCall, Timeout, Ntries), ?wait_match(Pattern, FunctionCall, ok, Timeout, Ntries)). -define(wait_match(Pattern, FunctionCall, Bind), ?wait_match(Pattern, FunctionCall, Bind, 500, 10) ). diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 004db6e3a2..6aaa22a6b4 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.6.4 +SSH_VSN = 4.6.5 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml index ea23cca2ee..33f29f38da 100644 --- a/lib/stdlib/doc/src/assert_hrl.xml +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -93,7 +93,7 @@ erlc -DNOASSERT=true *.erl</code> <taglist> <tag><c>assert(BoolExpr)</c></tag> <item></item> - <tag><c>URKAassert(BoolExpr, Comment)</c></tag> + <tag><c>assert(BoolExpr, Comment)</c></tag> <item> <p>Tests that <c>BoolExpr</c> completes normally returning <c>true</c>.</p> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 1b31a1ec9d..305376a425 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -487,6 +487,11 @@ Error: fun containing local Erlang function calls <p>The pid of the heir of the table, or <c>none</c> if no heir is set.</p> </item> + <tag><c>{id,</c><seealso marker="#type-tid"> + <c>tid()</c></seealso><c>}</c></tag> + <item> + <p>The table identifier.</p> + </item> <tag><c>{keypos, integer() >= 1}</c></tag> <item> <p>The key position.</p> @@ -1074,10 +1079,13 @@ ets:select(Table, MatchSpec),</code> </item> <tag><c>named_table</c></tag> <item> - <p>If this option is present, name <c><anno>Name</anno></c> is - associated with the table identifier. The name can then - be used instead of the table identifier in subsequent - operations.</p> + <p>If this option is present, the table is registered under its + <c><anno>Name</anno></c> which can then be used instead of the + table identifier in subsequent operations.</p> + <p>The function will also return the <c><anno>Name</anno></c> + instead of the table identifier. To get the table identifier of a + named table, use + <seealso marker="#whereis/1"><c>whereis/1</c></seealso>.</p> </item> <tag><c>{keypos,<anno>Pos</anno>}</c></tag> <item> @@ -2037,6 +2045,21 @@ true</pre> </list> </desc> </func> + + <func> + <name name="whereis" arity="1"/> + <fsummary>Retrieves the tid() of a named table.</fsummary> + <desc> + <p>This function returns the + <seealso marker="#type-tid"><c>tid()</c></seealso> of the named table + identified by <c><anno>TableName</anno></c>, or <c>undefined</c> if + no such table exists. The <c>tid()</c> can be used in place of the + table name in all operations, which is slightly faster since the name + does not have to be resolved on each call.</p> + <p>If the table is deleted, the <c>tid()</c> will be invalid even if + another named table is created with the same name.</p> + </desc> + </func> </funcs> </erlref> diff --git a/lib/stdlib/doc/src/uri_string.xml b/lib/stdlib/doc/src/uri_string.xml index 21f470e763..88d4600611 100644 --- a/lib/stdlib/doc/src/uri_string.xml +++ b/lib/stdlib/doc/src/uri_string.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2017</year><year>2017</year> + <year>2017</year><year>2018</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -24,7 +24,7 @@ <title>uri_string</title> <prepared>Péter Dimitrov</prepared> <docno>1</docno> - <date>2017-10-24</date> + <date>2018-02-07</date> <rev>A</rev> </header> <module>uri_string</module> @@ -32,7 +32,11 @@ <description> <p>This module contains functions for parsing and handling URIs (<url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>) and - form-urlencoded query strings (<url href="https://www.w3.org/TR/html5/forms.html">HTML5</url>). + form-urlencoded query strings (<url href="https://www.w3.org/TR/html52/">HTML 5.2</url>). + </p> + <p> + Parsing and serializing non-UTF-8 form-urlencoded query strings are also supported + (<url href="https://www.w3.org/TR/html50/">HTML 5.0</url>). </p> <p>A URI is an identifier consisting of a sequence of characters matching the syntax rule named <em>URI</em> in <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>. @@ -70,7 +74,8 @@ <seealso marker="#transcode/2"><c>transcode/2</c></seealso> </item> <item>Transforming URIs into a normalized form<br></br> - <seealso marker="#normalize/1"><c>normalize/1</c></seealso> + <seealso marker="#normalize/1"><c>normalize/1</c></seealso><br></br> + <seealso marker="#normalize/2"><c>normalize/2</c></seealso> </item> <item>Composing form-urlencoded query strings from a list of key-value pairs<br></br> <seealso marker="#compose_query/1"><c>compose_query/1</c></seealso><br></br> @@ -151,8 +156,10 @@ <p>Composes a form-urlencoded <c><anno>QueryString</anno></c> based on a <c><anno>QueryList</anno></c>, a list of non-percent-encoded key-value pairs. Form-urlencoding is defined in section - 4.10.22.6 of the <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - specification. + 4.10.21.6 of the <url href="https://www.w3.org/TR/html52/">HTML 5.2</url> + specification and in section 4.10.22.6 of the + <url href="https://www.w3.org/TR/html50/">HTML 5.0</url> specification for + non-UTF-8 encodings. </p> <p>See also the opposite operation <seealso marker="#dissect_query/1"> <c>dissect_query/1</c></seealso>. @@ -209,12 +216,11 @@ <p>Dissects an urlencoded <c><anno>QueryString</anno></c> and returns a <c><anno>QueryList</anno></c>, a list of non-percent-encoded key-value pairs. Form-urlencoding is defined in section - 4.10.22.6 of the <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - specification. + 4.10.21.6 of the <url href="https://www.w3.org/TR/html52/">HTML 5.2</url> + specification and in section 4.10.22.6 of the + <url href="https://www.w3.org/TR/html50/">HTML 5.0</url> specification for + non-UTF-8 encodings. </p> - <p>It is not as strict for its input as the decoding algorithm defined by - <url href="https://www.w3.org/TR/html5/forms.html">HTML5</url> - and accepts all unicode characters.</p> <p>See also the opposite operation <seealso marker="#compose_query/1"> <c>compose_query/1</c></seealso>. </p> @@ -233,7 +239,7 @@ <name name="normalize" arity="1"/> <fsummary>Syntax-based normalization.</fsummary> <desc> - <p>Transforms <c><anno>URIString</anno></c> into a normalized form + <p>Transforms an <c><anno>URI</anno></c> into a normalized form using Syntax-Based Normalization as defined by <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>.</p> <p>This function implements case normalization, percent-encoding @@ -247,6 +253,33 @@ <![CDATA[<<"mid/6">>]]> 3> uri_string:normalize("http://localhost:80"). "https://localhost/" +4> <input>uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g",</input> +4> host => "localhost-örebro"}). +"http://localhost-%C3%B6rebro/a/g" + </pre> + </desc> + </func> + + <func> + <name name="normalize" arity="2"/> + <fsummary>Syntax-based normalization.</fsummary> + <desc> + <p>Same as <c>normalize/1</c> but with an additional + <c><anno>Options</anno></c> parameter, that controls if the normalized URI + shall be returned as an uri_map(). + There is one supported option: <c>return_map</c>. + </p> + <p><em>Example:</em></p> + <pre> +1> <input>uri_string:normalize("/a/b/c/./../../g", [return_map]).</input> +#{path => "/a/g"} +2> <![CDATA[uri_string:normalize(<<"mid/content=5/../6">>, [return_map]).]]> +<![CDATA[#{path => <<"mid/6">>}]]> +3> uri_string:normalize("http://localhost:80", [return_map]). +#{scheme => "http",path => "/",host => "localhost"} +4> <input>uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g",</input> +4> host => "localhost-örebro"}, [return_map]). +#{scheme => "http",path => "/a/g",host => "localhost-örebro"} </pre> </desc> </func> diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 06c15fceda..24349c74e8 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -148,7 +148,8 @@ chunks(File, Chunks, Options) -> try read_chunk_data(File, Chunks, Options) catch Error -> Error end. --spec all_chunks(beam()) -> {'ok', 'beam_lib', [{chunkid(), dataB()}]}. +-spec all_chunks(beam()) -> + {'ok', 'beam_lib', [{chunkid(), dataB()}]} | {'error', 'beam_lib', info_rsn()}. all_chunks(File) -> read_all_chunks(File). diff --git a/lib/stdlib/src/binary.erl b/lib/stdlib/src/binary.erl index 6a64133b45..7d0e42489e 100644 --- a/lib/stdlib/src/binary.erl +++ b/lib/stdlib/src/binary.erl @@ -47,23 +47,39 @@ at(_, _) -> -spec bin_to_list(Subject) -> [byte()] when Subject :: binary(). -bin_to_list(_) -> - erlang:nif_error(undef). +bin_to_list(Subject) -> + binary_to_list(Subject). -spec bin_to_list(Subject, PosLen) -> [byte()] when Subject :: binary(), PosLen :: part(). -bin_to_list(_, _) -> - erlang:nif_error(undef). +bin_to_list(Subject, {Pos, Len}) -> + bin_to_list(Subject, Pos, Len); +bin_to_list(_Subject, _BadArg) -> + erlang:error(badarg). -spec bin_to_list(Subject, Pos, Len) -> [byte()] when Subject :: binary(), Pos :: non_neg_integer(), Len :: integer(). -bin_to_list(_, _, _) -> - erlang:nif_error(undef). +bin_to_list(Subject, Pos, Len) when not is_binary(Subject); + not is_integer(Pos); + not is_integer(Len) -> + %% binary_to_list/3 allows bitstrings as long as the slice fits, and we + %% want to badarg when Pos/Len aren't integers instead of raising badarith + %% when adjusting args for binary_to_list/3. + erlang:error(badarg); +bin_to_list(Subject, Pos, 0) when Pos >= 0, Pos =< byte_size(Subject) -> + %% binary_to_list/3 doesn't handle this case. + []; +bin_to_list(_Subject, _Pos, 0) -> + erlang:error(badarg); +bin_to_list(Subject, Pos, Len) when Len < 0 -> + bin_to_list(Subject, Pos + Len, -Len); +bin_to_list(Subject, Pos, Len) when Len > 0 -> + binary_to_list(Subject, Pos + 1, Pos + Len). -spec compile_pattern(Pattern) -> cp() when Pattern :: binary() | [binary()]. diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index 00e6a10d8a..77cc88eb08 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1197,21 +1197,21 @@ skip_else(_Else, From, St, Sis) -> %% macro_expansion(Tokens, Anno) %% Extract the macro parameters and the expansion from a macro definition. -macro_pars([{')',_Lp}, {',',Ld}|Ex], Args) -> - {ok, {lists:reverse(Args), macro_expansion(Ex, Ld)}}; -macro_pars([{var,_,Name}, {')',_Lp}, {',',Ld}|Ex], Args) -> +macro_pars([{')',_Lp}, {',',_Ld}=Comma|Ex], Args) -> + {ok, {lists:reverse(Args), macro_expansion(Ex, Comma)}}; +macro_pars([{var,_,Name}, {')',_Lp}, {',',_Ld}=Comma|Ex], Args) -> false = lists:member(Name, Args), %Prolog is nice - {ok, {lists:reverse([Name|Args]), macro_expansion(Ex, Ld)}}; + {ok, {lists:reverse([Name|Args]), macro_expansion(Ex, Comma)}}; macro_pars([{var,_L,Name}, {',',_}|Ts], Args) -> false = lists:member(Name, Args), macro_pars(Ts, [Name|Args]). -macro_expansion([{')',_Lp},{dot,_Ld}], _Anno0) -> []; -macro_expansion([{dot,_}=Dot], _Anno0) -> +macro_expansion([{')',_Lp},{dot,_Ld}], _T0) -> []; +macro_expansion([{dot,_}=Dot], _T0) -> throw({error,loc(Dot),missing_parenthesis}); -macro_expansion([T|Ts], _Anno0) -> +macro_expansion([T|Ts], _T0) -> [T|macro_expansion(Ts, T)]; -macro_expansion([], Anno0) -> throw({error,loc(Anno0),premature_end}). +macro_expansion([], T0) -> throw({error,loc(T0),premature_end}). %% expand_macros(Tokens, St) %% expand_macro(Tokens, MacroToken, RestTokens) diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index 039ab45868..6a559f0be5 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -73,7 +73,8 @@ select_count/2, select_delete/2, select_replace/2, select_reverse/1, select_reverse/2, select_reverse/3, setopts/2, slot/2, take/2, - update_counter/3, update_counter/4, update_element/3]). + update_counter/3, update_counter/4, update_element/3, + whereis/1]). %% internal exports -export([internal_request_all/0]). @@ -145,6 +146,7 @@ give_away(_, _, _) -> InfoList :: [InfoTuple], InfoTuple :: {compressed, boolean()} | {heir, pid() | none} + | {id, tid()} | {keypos, pos_integer()} | {memory, non_neg_integer()} | {name, atom()} @@ -162,7 +164,7 @@ info(_) -> -spec info(Tab, Item) -> Value | undefined when Tab :: tab(), - Item :: compressed | fixed | heir | keypos | memory + Item :: compressed | fixed | heir | id | keypos | memory | name | named_table | node | owner | protection | safe_fixed | safe_fixed_monotonic_time | size | stats | type | write_concurrency | read_concurrency, @@ -512,6 +514,11 @@ update_counter(_, _, _, _) -> update_element(_, _, _) -> erlang:nif_error(undef). +-spec whereis(TableName) -> tid() | undefined when + TableName :: atom(). +whereis(_) -> + erlang:nif_error(undef). + %%% End of BIFs -opaque comp_match_spec() :: reference(). diff --git a/lib/stdlib/src/uri_string.erl b/lib/stdlib/src/uri_string.erl index a84679c595..28d36ea229 100644 --- a/lib/stdlib/src/uri_string.erl +++ b/lib/stdlib/src/uri_string.erl @@ -227,7 +227,7 @@ %% External API %%------------------------------------------------------------------------- -export([compose_query/1, compose_query/2, - dissect_query/1, normalize/1, parse/1, + dissect_query/1, normalize/1, normalize/2, parse/1, recompose/1, transcode/2]). -export_type([error/0, uri_map/0, uri_string/0]). @@ -292,18 +292,36 @@ %%------------------------------------------------------------------------- %% Normalize URIs %%------------------------------------------------------------------------- --spec normalize(URIString) -> NormalizedURI when - URIString :: uri_string(), - NormalizedURI :: uri_string(). -normalize(URIString) -> - %% Percent-encoding normalization and case normalization for - %% percent-encoded triplets are achieved by running parse and - %% recompose on the input URI string. - recompose( - normalize_path_segment( - normalize_scheme_based( - normalize_case( - parse(URIString))))). +-spec normalize(URI) -> NormalizedURI when + URI :: uri_string() | uri_map(), + NormalizedURI :: uri_string() + | error(). +normalize(URIMap) -> + normalize(URIMap, []). + + +-spec normalize(URI, Options) -> NormalizedURI when + URI :: uri_string() | uri_map(), + Options :: [return_map], + NormalizedURI :: uri_string() | uri_map(). +normalize(URIMap, []) when is_map(URIMap) -> + recompose(normalize_map(URIMap)); +normalize(URIMap, [return_map]) when is_map(URIMap) -> + normalize_map(URIMap); +normalize(URIString, []) -> + case parse(URIString) of + Value when is_map(Value) -> + recompose(normalize_map(Value)); + Error -> + Error + end; +normalize(URIString, [return_map]) -> + case parse(URIString) of + Value when is_map(Value) -> + normalize_map(Value); + Error -> + Error + end. %%------------------------------------------------------------------------- @@ -385,7 +403,8 @@ transcode(URIString, Options) when is_list(URIString) -> %%------------------------------------------------------------------------- %% Functions for working with the query part of a URI as a list %% of key/value pairs. -%% HTML5 - 4.10.22.6 URL-encoded form data +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - non UTF-8 %%------------------------------------------------------------------------- %%------------------------------------------------------------------------- @@ -393,7 +412,7 @@ transcode(URIString, Options) when is_list(URIString) -> %% (application/x-www-form-urlencoded encoding algorithm) %%------------------------------------------------------------------------- -spec compose_query(QueryList) -> QueryString when - QueryList :: [{uri_string(), uri_string()}], + QueryList :: [{unicode:chardata(), unicode:chardata()}], QueryString :: uri_string() | error(). compose_query(List) -> @@ -401,7 +420,7 @@ compose_query(List) -> -spec compose_query(QueryList, Options) -> QueryString when - QueryList :: [{uri_string(), uri_string()}], + QueryList :: [{unicode:chardata(), unicode:chardata()}], Options :: [{encoding, atom()}], QueryString :: uri_string() | error(). @@ -432,7 +451,7 @@ compose_query([], _Options, IsList, Acc) -> %%------------------------------------------------------------------------- -spec dissect_query(QueryString) -> QueryList when QueryString :: uri_string(), - QueryList :: [{uri_string(), uri_string()}] + QueryList :: [{unicode:chardata(), unicode:chardata()}] | error(). dissect_query(<<>>) -> []; @@ -1755,7 +1774,8 @@ get_separator(_L) -> <<"&">>. -%% HTML5 - 4.10.22.6 URL-encoded form data - encoding +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - encoding (non UTF-8) form_urlencode(Cs, [{encoding, latin1}]) when is_list(Cs) -> B = convert_to_binary(Cs, utf8, utf8), html5_byte_encode(base10_encode(B)); @@ -1850,7 +1870,8 @@ dissect_query_value(<<>>, IsList, Acc, Key, Value) -> lists:reverse([{K,V}|Acc]). -%% Form-urldecode input based on RFC 1866 [8.2.1] +%% HTML 5.2 - 4.10.21.6 URL-encoded form data - WHATWG URL (10 Jan 2018) - UTF-8 +%% HTML 5.0 - 4.10.22.6 URL-encoded form data - decoding (non UTF-8) form_urldecode(true, B) -> Result = base10_decode(form_urldecode(B, <<>>)), convert_to_list(Result, utf8); @@ -1903,6 +1924,12 @@ base10_decode_unicode(<<H,_/binary>>, _, _) -> %% Helper functions for normalize %%------------------------------------------------------------------------- +normalize_map(URIMap) -> + normalize_path_segment( + normalize_scheme_based( + normalize_case(URIMap))). + + %% 6.2.2.1. Case Normalization normalize_case(#{scheme := Scheme, host := Host} = Map) -> Map#{scheme => to_lower(Scheme), diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 0cfb1b5516..8b651f4b43 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -78,6 +78,7 @@ -export([ets_all/1]). -export([massive_ets_all/1]). -export([take/1]). +-export([whereis_table/1]). -export([init_per_testcase/2, end_per_testcase/2]). %% Convenience for manual testing @@ -137,7 +138,8 @@ all() -> otp_9423, ets_all, massive_ets_all, - take]. + take, + whereis_table]. groups() -> [{new, [], @@ -4099,6 +4101,7 @@ info_do(Opts) -> {value, {keypos, 2}} = lists:keysearch(keypos, 1, Res), {value, {protection, protected}} = lists:keysearch(protection, 1, Res), + {value, {id, Tab}} = lists:keysearch(id, 1, Res), true = ets:delete(Tab), undefined = ets:info(non_existing_table_xxyy), undefined = ets:info(non_existing_table_xxyy,type), @@ -5892,6 +5895,36 @@ take(Config) when is_list(Config) -> ets:delete(T3), ok. +whereis_table(Config) when is_list(Config) -> + %% Do we return 'undefined' when the named table doesn't exist? + undefined = ets:whereis(whereis_test), + + %% Does the tid() refer to the same table as the name? + whereis_test = ets:new(whereis_test, [named_table]), + Tid = ets:whereis(whereis_test), + + ets:insert(whereis_test, [{hello}, {there}]), + + [[{hello}],[{there}]] = ets:match(whereis_test, '$1'), + [[{hello}],[{there}]] = ets:match(Tid, '$1'), + + true = ets:delete_all_objects(Tid), + + [] = ets:match(whereis_test, '$1'), + [] = ets:match(Tid, '$1'), + + %% Does the name disappear when deleted through the tid()? + true = ets:delete(Tid), + undefined = ets:info(whereis_test), + {'EXIT',{badarg, _}} = (catch ets:match(whereis_test, '$1')), + + %% Is the old tid() broken when the table is re-created with the same + %% name? + whereis_test = ets:new(whereis_test, [named_table]), + [] = ets:match(whereis_test, '$1'), + {'EXIT',{badarg, _}} = (catch ets:match(Tid, '$1')), + + ok. %% %% Utility functions: diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 15ae4fa2d7..d753d929f5 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2017. All Rights Reserved. +%% Copyright Ericsson AB 2000-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -52,7 +52,9 @@ all() -> [seed, interval_int, interval_float, api_eq, reference, - {group, basic_stats}, uniform_real_conv, + {group, basic_stats}, + {group, distr_stats}, + uniform_real_conv, plugin, measure, {group, reference_jump} ]. @@ -60,8 +62,9 @@ all() -> groups() -> [{basic_stats, [parallel], [basic_stats_uniform_1, basic_stats_uniform_2, - basic_stats_standard_normal, - stats_standard_normal_box_muller, + basic_stats_standard_normal]}, + {distr_stats, [parallel], + [stats_standard_normal_box_muller, stats_standard_normal_box_muller_2, stats_standard_normal]}, {reference_jump, [parallel], @@ -70,6 +73,9 @@ groups() -> group(basic_stats) -> %% valgrind needs a lot of time [{timetrap,{minutes,10}}]; +group(distr_stats) -> + %% valgrind needs a lot of time + [{timetrap,{minutes,10}}]; group(reference_jump) -> %% valgrind needs a lot of time [{timetrap,{minutes,10}}]. @@ -437,7 +443,7 @@ stats_standard_normal_box_muller(Config) when is_list(Config) -> {Z, [S]} end, State = [rand:seed(exrop)], - stats_standard_normal(NormalS, State) + stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -462,7 +468,7 @@ stats_standard_normal_box_muller_2(Config) when is_list(Config) -> {Z, [S]} end, State = [rand:seed(exrop)], - stats_standard_normal(NormalS, State) + stats_standard_normal(NormalS, State, 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. @@ -472,21 +478,21 @@ stats_standard_normal(Config) when is_list(Config) -> try math:erfc(1.0) of _ -> stats_standard_normal( - fun rand:normal_s/1, rand:seed_s(exrop)) + fun rand:normal_s/1, rand:seed_s(exrop), 3) catch error:_ -> {skip, "math:erfc/1 not supported"} end. %% -stats_standard_normal(Fun, S) -> +stats_standard_normal(Fun, S, Retries) -> %%% %%% ct config: -%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.2}]}]}. +%%% {rand_SUITE, [{stats_standard_normal,[{seconds, 8}, {std_devs, 4.0}]}]}. %%% Seconds = ct:get_config({?MODULE, ?FUNCTION_NAME, seconds}, 8), StdDevs = ct:get_config( {?MODULE, ?FUNCTION_NAME, std_devs}, - 4.2), % probability erfc(4.2/sqrt(2)) (1/37465) to fail a bucket + 4.0), % probability erfc(4.0/sqrt(2)) (1/15787) to fail a bucket %%% ct:timetrap({seconds, Seconds + 120}), %% Buckets is chosen to get a range where the the probability to land @@ -505,11 +511,11 @@ stats_standard_normal(Fun, S) -> P0 = math:erf(1 / W), Rounds = TargetHits * ceil(1.0 / P0), Histogram = array:new({default, 0}), - StopTime = erlang:monotonic_time(second) + Seconds, ct:pal( "Running standard normal test against ~w std devs for ~w seconds...", [StdDevs, Seconds]), - {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds} = + StopTime = erlang:monotonic_time(second) + Seconds, + {PositiveHistogram, NegativeHistogram, Outlier, TotalRounds, NewS} = stats_standard_normal( InvDelta, Buckets, Histogram, Histogram, 0.0, Fun, S, Rounds, StopTime, Rounds, 0), @@ -522,16 +528,33 @@ stats_standard_normal(Fun, S) -> "Total rounds: ~w, tolerance: 1/~.2f..1/~.2f, " "outlier: ~.2f, probability 1/~.2f.", [TotalRounds, Precision, TopPrecision, Outlier, InvOP]), - {TotalRounds, [], []} = - {TotalRounds, + case + {bucket_error, TotalRounds, check_histogram( W, TotalRounds, StdDevs, PositiveHistogram, Buckets), check_histogram( - W, TotalRounds, StdDevs, NegativeHistogram, Buckets)}, - %% If the probability for getting this Outlier is lower than 1/50, - %% then this is fishy! - true = (1/50 =< OutlierProbability), - {comment, {tp, TopPrecision, op, InvOP}}. + W, TotalRounds, StdDevs, NegativeHistogram, Buckets)} + of + {_, _, [], []} when InvOP < 100 -> + {comment, {tp, TopPrecision, op, InvOP}}; + {_, _, [], []} -> + %% If the probability for getting this Outlier is lower than + %% 1/100, then this is fishy! + stats_standard_normal( + Fun, NewS, Retries, {outlier_fishy, InvOP}); + BucketErrors -> + stats_standard_normal( + Fun, NewS, Retries, BucketErrors) + end. +%% +stats_standard_normal(Fun, S, Retries, Failure) -> + case Retries - 1 of + 0 -> + ct:fail(Failure); + NewRetries -> + ct:pal("Retry due to TC glitch: ~p", [Failure]), + stats_standard_normal(Fun, S, NewRetries) + end. %% stats_standard_normal( InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, @@ -544,7 +567,7 @@ stats_standard_normal( Fun, S, Rounds, StopTime, Rounds, TotalRounds + Rounds); _ -> {PositiveHistogram, NegativeHistogram, - Outlier, TotalRounds + Rounds} + Outlier, TotalRounds + Rounds, S} end; stats_standard_normal( InvDelta, Buckets, PositiveHistogram, NegativeHistogram, Outlier, @@ -571,9 +594,6 @@ increment_bucket(Bucket, Array) -> array:set(Bucket, array:get(Bucket, Array) + 1, Array). check_histogram(W, Rounds, StdDevs, Histogram, Buckets) -> - %%PrevBucket = 512, - %%Bucket = PrevBucket - 1, - %%P = 0.5 * math:erfc(PrevBucket / W), TargetP = 0.5 * math:erfc(Buckets / W), P = 0.0, N = 0, @@ -592,7 +612,7 @@ check_histogram( P = 0.5 * math:erfc(Bucket / W), BucketP = P - PrevP, if - TargetP =< BucketP -> + BucketP < TargetP -> check_histogram( W, Rounds, StdDevs, Histogram, TargetP, Bucket - 1, PrevBucket, PrevP, N); @@ -604,7 +624,7 @@ check_histogram( UpperLimit = ceil(Exp + Threshold), if N < LowerLimit; UpperLimit < N -> - [#{bucket => {Bucket, PrevBucket}, n => N, exp => Exp, + [#{bucket => {Bucket, PrevBucket}, n => N, lower => LowerLimit, upper => UpperLimit} | check_histogram( W, Rounds, StdDevs, Histogram, TargetP, diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index 71f86e32e5..7b82647416 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -894,10 +894,13 @@ match_limit(Config) when is_list(Config) -> %% Test that we get sub-binaries if subject is a binary and we capture %% binaries. sub_binaries(Config) when is_list(Config) -> - Bin = list_to_binary(lists:seq(1,255)), - {match,[B,C]}=re:run(Bin,"(a)",[{capture,all,binary}]), - 255 = binary:referenced_byte_size(B), - 255 = binary:referenced_byte_size(C), - {match,[D]}=re:run(Bin,"(a)",[{capture,[1],binary}]), - 255 = binary:referenced_byte_size(D), + %% The GC can auto-convert tiny sub-binaries to heap binaries, so we + %% extract large sequences to make the test more stable. + Bin = << <<I>> || I <- lists:seq(1, 4096) >>, + {match,[B,C]}=re:run(Bin,"a(.+)$",[{capture,all,binary}]), + true = byte_size(B) =/= byte_size(C), + 4096 = binary:referenced_byte_size(B), + 4096 = binary:referenced_byte_size(C), + {match,[D]}=re:run(Bin,"a(.+)$",[{capture,[1],binary}]), + 4096 = binary:referenced_byte_size(D), ok. diff --git a/lib/stdlib/test/uri_string_SUITE.erl b/lib/stdlib/test/uri_string_SUITE.erl index fef356355c..92f8bb3292 100644 --- a/lib/stdlib/test/uri_string_SUITE.erl +++ b/lib/stdlib/test/uri_string_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0,groups/0, - normalize/1, + normalize/1, normalize_map/1, normalize_return_map/1, normalize_negative/1, parse_binary_fragment/1, parse_binary_host/1, parse_binary_host_ipv4/1, parse_binary_host_ipv6/1, parse_binary_path/1, parse_binary_pct_encoded_fragment/1, parse_binary_pct_encoded_query/1, @@ -68,6 +68,9 @@ suite() -> all() -> [ normalize, + normalize_map, + normalize_return_map, + normalize_negative, parse_binary_scheme, parse_binary_userinfo, parse_binary_pct_encoded_userinfo, @@ -912,6 +915,56 @@ normalize(_Config) -> <<"tftp://localhost">> = uri_string:normalize(<<"tftp://localhost:69">>). +normalize_map(_Config) -> + "/a/g" = uri_string:normalize(#{path => "/a/b/c/./../../g"}), + <<"mid/6">> = uri_string:normalize(#{path => <<"mid/content=5/../6">>}), + "http://localhost-%C3%B6rebro/a/g" = + uri_string:normalize(#{scheme => "http",port => 80,path => "/a/b/c/./../../g", + host => "localhost-örebro"}), + <<"http://localhost-%C3%B6rebro/a/g">> = + uri_string:normalize(#{scheme => <<"http">>,port => 80, + path => <<"/a/b/c/./../../g">>, + host => <<"localhost-örebro"/utf8>>}), + <<"https://localhost/">> = + uri_string:normalize(#{scheme => <<"https">>,port => 443,path => <<>>, + host => <<"localhost">>}), + <<"https://localhost:445/">> = + uri_string:normalize(#{scheme => <<"https">>,port => 445,path => <<>>, + host => <<"localhost">>}), + <<"ftp://localhost">> = + uri_string:normalize(#{scheme => <<"ftp">>,port => 21,path => <<>>, + host => <<"localhost">>}), + <<"ssh://localhost">> = + uri_string:normalize(#{scheme => <<"ssh">>,port => 22,path => <<>>, + host => <<"localhost">>}), + <<"sftp://localhost">> = + uri_string:normalize(#{scheme => <<"sftp">>,port => 22,path => <<>>, + host => <<"localhost">>}), + <<"tftp://localhost">> = + uri_string:normalize(#{scheme => <<"tftp">>,port => 69,path => <<>>, + host => <<"localhost">>}). + +normalize_return_map(_Config) -> + #{scheme := "http",path := "/a/g",host := "localhost-örebro"} = + uri_string:normalize("http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g", + [return_map]), + #{scheme := <<"http">>,path := <<"/a/g">>, host := <<"localhost-örebro"/utf8>>} = + uri_string:normalize(<<"http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g">>, + [return_map]), + #{scheme := <<"https">>,path := <<"/">>, host := <<"localhost">>} = + uri_string:normalize(#{scheme => <<"https">>,port => 443,path => <<>>, + host => <<"localhost">>}, [return_map]). + +normalize_negative(_Config) -> + {error,invalid_uri,":"} = + uri_string:normalize("http://local>host"), + {error,invalid_uri,":"} = + uri_string:normalize(<<"http://local>host">>), + {error,invalid_uri,":"} = + uri_string:normalize("http://[192.168.0.1]", [return_map]), + {error,invalid_uri,":"} = + uri_string:normalize(<<"http://[192.168.0.1]">>, [return_map]). + interop_query_utf8(_Config) -> Q = uri_string:compose_query([{"foo bar","1"}, {"合", "2"}]), Uri = uri_string:recompose(#{path => "/", query => Q}), diff --git a/otp_versions.table b/otp_versions.table index 5626fdff88..6b95fd5c80 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-20.2.4 : ssh-4.6.5 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2.1 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.2 megaco-3.18.3 mnesia-4.15.3 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.4 sasl-3.1.1 snmp-5.2.9 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : OTP-20.2.3 : erts-9.2.1 kernel-5.4.2 runtime_tools-1.12.4 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 megaco-3.18.3 mnesia-4.15.3 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 sasl-3.1.1 snmp-5.2.9 ssh-4.6.4 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : OTP-20.2.2 : mnesia-4.15.3 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.1 megaco-3.18.3 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.3 sasl-3.1.1 snmp-5.2.9 ssh-4.6.4 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : OTP-20.2.1 : ssh-4.6.4 # asn1-5.0.4 common_test-1.15.3 compiler-7.1.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2 debugger-4.2.4 dialyzer-3.2.3 diameter-2.1.3 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.2 erl_interface-3.10.1 erts-9.2 et-1.6.1 eunit-2.3.5 hipe-3.17 ic-4.4.3 inets-6.4.5 jinterface-1.8.1 kernel-5.4.1 megaco-3.18.3 mnesia-4.15.2 observer-2.6 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.3 sasl-3.1.1 snmp-5.2.9 ssl-8.2.3 stdlib-3.4.3 syntax_tools-2.1.4 tools-2.11.1 wx-1.8.3 xmerl-1.3.16 : diff --git a/system/doc/efficiency_guide/advanced.xml b/system/doc/efficiency_guide/advanced.xml index 896eda5f1c..245e7d3674 100644 --- a/system/doc/efficiency_guide/advanced.xml +++ b/system/doc/efficiency_guide/advanced.xml @@ -151,7 +151,7 @@ <row> <cell>Processes</cell> <cell>The maximum number of simultaneously alive Erlang processes - is by default 32,768. This limit can be configured at startup. + is by default 262,144. This limit can be configured at startup. For more information, see the <seealso marker="erts:erl#max_processes"><c>+P</c></seealso> command-line flag in the |