diff options
90 files changed, 2090 insertions, 1028 deletions
diff --git a/.travis.yml b/.travis.yml index ef17d6fbe7..42151a16d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,10 +33,8 @@ script: after_success: - $ERL_TOP/bin/dialyzer --build_plt --apps asn1 compiler crypto dialyzer edoc erts et hipe inets kernel mnesia observer public_key runtime_tools snmp ssh ssl stdlib syntax_tools wx xmerl --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts kernel stdlib --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps asn1 crypto dialyzer --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps hipe parsetools public_key --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps runtime_tools sasl tools --statistics + - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts kernel stdlib asn1 crypto dialyzer hipe parsetools public_key runtime_tools sasl tools --statistics + - $ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc inets mnesia observer ssh ssl syntax_tools tools wx xmerl --statistics - ./otp_build tests && make release_docs after_script: diff --git a/OTP_VERSION b/OTP_VERSION index a824897d28..34716091d4 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -19.2 +19.2.1 @@ -40,7 +40,7 @@ In short: * Go to the JIRA issue tracker at [bugs.erlang.org] [7] to see reported issues which you can contribute to. - Search for issues with the status *Contribution Needed*. + Search for issues with the status *Help Wanted*. Bug Reports diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 11777f0014..812538729d 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -32,6 +32,26 @@ <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 8.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix a quite rare bug causing VM crash during code loading + and the use of export funs (fun M:F/A) of not yet loaded + modules. Requires a very specfic timing of concurrent + scheduler threads. Has been seen on ARM but can probably + also occure on other architectures. Bug has existed since + OTP R16.</p> + <p> + Own Id: OTP-14144 Aux Id: seq13242 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 644fd9fbf9..7f5e9ff691 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -786,7 +786,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } if (BIF_ARG_2 == am_true) { - int i; + int i, num_exps; /* * Make the code with the on_load function current. @@ -802,7 +802,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) /* * The on_load function succeded. Fix up export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->code[0] != BIF_ARG_1) { continue; @@ -822,14 +823,15 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { - int i; + int i, num_exps; /* * The on_load function failed. Remove references to the * code that is about to be purged from the export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->code[0] != BIF_ARG_1) { continue; @@ -995,6 +997,12 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) goto literal_gc; *redsp += 1; + if (c_p->abandoned_heap) { + if (any_heap_refs(c_p->abandoned_heap, c_p->abandoned_heap + c_p->heap_sz, + literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + } if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) goto literal_gc; @@ -1295,6 +1303,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls #endif if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) goto try_literal_gc; + if (rp->abandoned_heap) { + if (any_heap_refs(rp->abandoned_heap, rp->abandoned_heap + rp->heap_sz, + literals, lit_bsize)) + goto try_literal_gc; + } if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) goto try_literal_gc; @@ -2022,9 +2035,9 @@ delete_code(Module* modp) { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = make_atom(modp->module); - int i; + int i, num_exps = export_list_size(code_ix); - for (i = 0; i < export_list_size(code_ix); i++) { + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->code[0] == module)) { if (ep->addressv[code_ix] == ep->code+3) { diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index bbb2e4f34f..0df2df0eaa 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -74,6 +74,9 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; +#ifdef ERTS_DIRTY_SCHEDULERS +erts_smp_mtx_t erts_dirty_bp_ix_mtx; +#endif /* * Inlined helpers @@ -85,6 +88,31 @@ get_mtime(Process *c_p) return erts_get_monotonic_time(erts_proc_sched_data(c_p)); } +static ERTS_INLINE Uint32 +acquire_bp_sched_ix(Process *c_p) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); + return (Uint32) erts_no_schedulers; + } +#endif + return (Uint32) esdp->no - 1; +} + +static ERTS_INLINE void +release_bp_sched_ix(Uint32 ix) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (ix == (Uint32) erts_no_schedulers) + erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); +#endif +} + + + /* ************************************************************************* ** Local prototypes */ @@ -135,6 +163,9 @@ void erts_bp_init(void) { erts_smp_atomic32_init_nob(&erts_active_bp_index, 0); erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index"); +#endif } @@ -973,6 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(c_p); ASSERT(c_p); ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING @@ -980,7 +1012,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* get previous timestamp and breakpoint * from the process psd */ - + pbt = ERTS_PROC_GET_CALL_TIME(c_p); time = get_mtime(c_p); @@ -1006,7 +1038,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* if null then the breakpoint was removed */ if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(c_p)]); + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1027,7 +1059,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* this breakpoint */ ASSERT(bdt); - h = &(bdt->hash[bp_sched2ix_proc(c_p)]); + h = &(bdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1041,6 +1073,8 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) pbt->pc = I; pbt->time = time; + + release_bp_sched_ix(six); } void @@ -1051,6 +1085,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING @@ -1071,6 +1106,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) */ if (pbt) { + /* might have been removed due to * trace_pattern(false) */ @@ -1085,7 +1121,8 @@ erts_trace_time_return(Process *p, BeamInstr *pc) /* beware, the trace_pattern might have been removed */ if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1096,11 +1133,15 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } else { BP_TIME_ADD(item, &sitem); } + } pbt->pc = pc; pbt->time = time; + } + + release_bp_sched_ix(six); } int @@ -1353,6 +1394,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); @@ -1375,7 +1417,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { sitem.pid = p->common.id; sitem.count = 0; - h = &(pbdt->hash[bp_sched2ix_proc(p)]); + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1401,6 +1443,8 @@ void erts_schedule_time_break(Process *p, Uint schedule) { break; } } /* pbt */ + + release_bp_sched_ix(six); } /* ************************************************************************* @@ -1517,7 +1561,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); - bdt->n = erts_no_total_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + bdt->n = erts_no_schedulers + 1; +#else + bdt->n = erts_no_schedulers; +#endif bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 7206ef471a..4743e4fc2f 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -79,10 +79,8 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) -#ifdef ERTS_SMP -#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1) -#else -#define bp_sched2ix_proc(p) (0) +#ifdef ERTS_DIRTY_SCHEDULERS +extern erts_smp_mtx_t erts_dirty_bp_ix_mtx; #endif enum erts_break_op{ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 66bccedd94..f392feb06b 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3552,18 +3552,11 @@ do { \ BifFunction vbf; ErlHeapFragment *live_hf_end; - if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the nif */ - goto context_switch; - } - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); c_p->current = I-3; /* current and vbf set to please handle_error */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; + HEAVY_SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = I[-1]; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0afdedf6c2..3f2bdf3f9d 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -797,14 +797,14 @@ erts_finish_loading(Binary* magic, Process* c_p, } else { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = stp->module; - int i; + int i, num_exps; /* * There is an -on_load() function. We will keep the current * code, but we must turn off any tracing. */ - - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep == NULL || ep->code[0] != module) { continue; @@ -5754,12 +5754,13 @@ exported_from_module(Process* p, /* Process whose heap to use. */ ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { - int i; + int i, num_exps; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export* ep = export_list(i,code_ix); if (ep->code[0] == mod) { diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index dfbe1ced47..61907cc7ff 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -717,6 +717,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * We have to be very very careful when doing this as the schedulers * could be anywhere. */ + sys_init_suspend_handler(); + for (i = 0; i < erts_no_schedulers; i++) { erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; if (!erts_equal_tids(tid,erts_thr_self())) diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 96275eb228..0627526d7e 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1062,9 +1062,16 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_smp_thr_progress_block(); } #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); +#endif + r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); +#endif #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { erts_smp_thr_progress_unblock(); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index cb48b31b7e..edbdbb4b8f 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -59,6 +59,10 @@ # define ERTS_GC_ASSERT(B) ((void) 1) #endif +#if defined(DEBUG) && 0 +# define HARDDEBUG 1 +#endif + /* * Returns number of elements in an array. */ @@ -110,7 +114,6 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static void remove_message_buffers(Process* p); static Eterm *full_sweep_heaps(Process *p, int hibernate, Eterm *n_heap, Eterm* n_htop, @@ -153,7 +156,7 @@ static void init_gc_info(ErtsGCInfo *gcip); static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz); #ifdef HARDDEBUG -static void disallow_heap_frag_ref_in_heap(Process* p); +static void disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop); static void disallow_heap_frag_ref_in_old_heap(Process* p); #endif @@ -489,24 +492,26 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE; hfrag = new_message_buffer(hsz); - hfrag->next = p->mbuf; - p->mbuf = hfrag; - p->mbuf_sz += hsz; p->heap = p->htop = &hfrag->mem[0]; p->hend = hend = &hfrag->mem[hsz]; p->stop = stop = hend - ssz; sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm)); if (p->abandoned_heap) { - /* Active heap already in a fragment; adjust it... */ - ErlHeapFragment *hfrag = ((ErlHeapFragment *) - (((char *) orig_heap) - - offsetof(ErlHeapFragment, mem))); - Uint unused = orig_hend - orig_htop; - ASSERT(hfrag->used_size == hfrag->alloc_size); - ASSERT(hfrag->used_size >= unused); - hfrag->used_size -= unused; - p->mbuf_sz -= unused; + /* + * Active heap already in a fragment; adjust it and + * save it into mbuf list... + */ + ErlHeapFragment *hfrag = ((ErlHeapFragment *) + (((char *) orig_heap) + - offsetof(ErlHeapFragment, mem))); + Uint used = orig_htop - orig_heap; + hfrag->used_size = used; + p->mbuf_sz += used; + ASSERT(hfrag->used_size <= hfrag->alloc_size); + ASSERT(!hfrag->off_heap.first && !hfrag->off_heap.overhead); + hfrag->next = p->mbuf; + p->mbuf = hfrag; } else { /* Do not leave a hole in the abandoned heap... */ @@ -568,17 +573,14 @@ young_gen_usage(Process *p) } } + hsz += p->htop - p->heap; aheap = p->abandoned_heap; - if (!aheap) - hsz += p->htop - p->heap; - else { + if (aheap) { /* used in orig heap */ if (p->flags & F_ABANDONED_HEAP_USE) hsz += aheap[p->heap_sz-1]; else hsz += p->heap_sz; - /* Remove unused part in latest fragment */ - hsz -= p->hend - p->htop; } return hsz; } @@ -614,7 +616,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, Uint reclaimed_now = 0; Eterm gc_trace_end_tag; int reds; - ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ + ErtsMonotonicTime start_time; ErtsSchedulerData *esdp; erts_aint32_t state; ERTS_MSACC_PUSH_STATE_M(); @@ -622,6 +624,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ERTS_UNDEF(start_time, 0); ERTS_CHK_MBUF_SZ(p); ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) @@ -788,6 +791,7 @@ do_major_collection: ASSERT(!p->mbuf); ASSERT(!ERTS_IS_GC_DESIRED(p)); + ASSERT(need <= HEAP_LIMIT(p) - HEAP_TOP(p)); return reds; } @@ -812,6 +816,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) >= erts_proc_sched_data(p)->virtual_reds); } + /* * Place all living data on a the new heap; deallocate any old heap. * Meant to be used by hibernate/3. @@ -858,11 +863,11 @@ erts_garbage_collect_hibernate(Process* p) p->arg_reg, p->arity); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : p->heap), - p->heap_sz * sizeof(Eterm)); +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, heap, htop); +#endif + + erts_deallocate_young_generation(p); p->heap = heap; p->high_water = htop; @@ -897,8 +902,6 @@ erts_garbage_collect_hibernate(Process* p) sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); - remove_message_buffers(p); - p->stop = p->hend = heap + heap_size; offs = heap - p->heap; @@ -1517,22 +1520,16 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, } #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : HEAP_START(p)), - HEAP_SIZE(p) * sizeof(Eterm)); - p->abandoned_heap = NULL; - p->flags &= ~F_ABANDONED_HEAP_USE; +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); +#endif + + erts_deallocate_young_generation(p); + HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; HEAP_END(p) = n_heap + new_sz; - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); } /* @@ -1621,13 +1618,12 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, } #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : HEAP_START(p)), - p->heap_sz * sizeof(Eterm)); - p->abandoned_heap = NULL; - p->flags &= ~F_ABANDONED_HEAP_USE; +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); +#endif + + erts_deallocate_young_generation(p); + HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; @@ -1636,11 +1632,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); - if (p->flags & F_ON_HEAP_MSGQ) move_msgq_to_heap(p); @@ -1793,22 +1784,56 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) return adjusted; } -/* - * Remove all message buffers. - */ -static void -remove_message_buffers(Process* p) +void +erts_deallocate_young_generation(Process *c_p) { - if (MBUF(p) != NULL) { - free_message_buffer(MBUF(p)); - MBUF(p) = NULL; + Eterm *orig_heap; + + if (!c_p->abandoned_heap) { + orig_heap = c_p->heap; + ASSERT(!(c_p->flags & F_ABANDONED_HEAP_USE)); + } + else { + ErlHeapFragment *hfrag; + + orig_heap = c_p->abandoned_heap; + c_p->abandoned_heap = NULL; + c_p->flags &= ~F_ABANDONED_HEAP_USE; + + /* + * Temporary heap located in heap fragment + * only referred to by 'c_p->heap'. Add it to + * 'c_p->mbuf' list and deallocate it as any + * other heap fragment... + */ + hfrag = ((ErlHeapFragment *) + (((char *) c_p->heap) + - offsetof(ErlHeapFragment, mem))); + + ASSERT(!hfrag->off_heap.first); + ASSERT(!hfrag->off_heap.overhead); + ASSERT(!hfrag->next); + ASSERT(c_p->htop - c_p->heap <= hfrag->alloc_size); + + hfrag->next = c_p->mbuf; + c_p->mbuf = hfrag; + } + + if (c_p->mbuf) { + free_message_buffer(c_p->mbuf); + c_p->mbuf = NULL; } - if (p->msg_frag) { - erts_cleanup_messages(p->msg_frag); - p->msg_frag = NULL; + if (c_p->msg_frag) { + erts_cleanup_messages(c_p->msg_frag); + c_p->msg_frag = NULL; } - MBUF_SIZE(p) = 0; + c_p->mbuf_sz = 0; + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, + orig_heap, + c_p->heap_sz * sizeof(Eterm)); } + #ifdef HARDDEBUG /* @@ -1820,19 +1845,15 @@ remove_message_buffers(Process* p) */ static void -disallow_heap_frag_ref_in_heap(Process* p) +disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop) { Eterm* hp; - Eterm* htop; - Eterm* heap; Uint heap_size; if (p->mbuf == 0) { return; } - htop = p->htop; - heap = p->heap; heap_size = (htop - heap)*sizeof(Eterm); hp = heap; @@ -3332,13 +3353,15 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) ErtsMessage* mp; Eterm *htop, *heap; - if (p->abandoned_heap) + if (p->abandoned_heap) { ERTS_GET_ORIG_HEAP(p, heap, htop); - else { - heap = p->heap; - htop = real_htop ? real_htop : HEAP_TOP(p); + if (heap <= ptr && ptr < htop) + return 1; } + heap = p->heap; + htop = real_htop ? real_htop : HEAP_TOP(p); + if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) { return 1; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 54ea9ca3c0..2521379664 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -27,10 +27,6 @@ #include "erl_map.h" -#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF -# define HARDDEBUG 1 -#endif - #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) @@ -157,5 +153,6 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); int erts_max_heap_size(Eterm, Uint *, Uint *); +void erts_deallocate_young_generation(Process *c_p); #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 38bebc7576..647fa26811 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2865,7 +2865,8 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) if (tmr->timeout <= btmp->now) left = 0; - left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + else + left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) ? tmr->receiver.name diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 2fd97208cc..070cc3f2d0 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2220,7 +2220,6 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); - sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 13a4b2cd93..5d4823c077 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -130,6 +130,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { #ifdef ERTS_DIRTY_SCHEDULERS { "dirty_run_queue_sleep_list", "address" }, { "dirty_gc_info", NULL }, + { "dirty_break_point_index", NULL }, #endif { "process_table", NULL }, { "cpu_info", NULL }, diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index e4c696ae3b..f45e6974cd 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -193,7 +193,7 @@ free_message_buffer(ErlHeapFragment* bp) erts_cleanup_offheap(&bp->off_heap); ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); bp = next_bp; }while (bp != NULL); } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6b265a8b80..19ce0f6965 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3561,8 +3561,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, #endif if (p) { /* This is almost a normal nif call like in beam_emu, - except that any heap fragment created in the nif will be - discarded without checking if anything in it is live. + except that any heap consumed by the nif will be + released without checking if anything in it is live. This is because we cannot do a GC here as we don't know the number of live registers that have to be preserved. This means that any heap part of the returned term may diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index cd0a2b2a21..41741764e8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8642,8 +8642,15 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, * from being selected for normal execution regardless * of locks held or not held on it... */ - ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS) - & erts_smp_atomic32_read_nob(&rp->state))); +#ifdef DEBUG + { + erts_aint32_t state; + state = erts_smp_atomic32_read_nob(&rp->state); + ASSERT((state & ERTS_PSFLG_PENDING_EXIT) + || !(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + } +#endif if (!suspend) resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); @@ -11962,7 +11969,6 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - Eterm *heap; ErtsPSD *psd; struct saved_calls *scb; process_breakpoint_time_t *pbt; @@ -12017,13 +12023,8 @@ delete_process(Process* p) hipe_delete_process(&p->hipe); #endif - heap = p->abandoned_heap ? p->abandoned_heap : p->heap; - -#ifdef DEBUG - sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif + erts_deallocate_young_generation(p); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -12035,16 +12036,6 @@ delete_process(Process* p) (p->old_hend-p->old_heap)*sizeof(Eterm)); } - /* - * Free all pending message buffers. - */ - if (p->mbuf != NULL) { - free_message_buffer(p->mbuf); - } - - if (p->msg_frag) - erts_cleanup_messages(p->msg_frag); - erts_erase_dicts(p); /* free all pending messages */ diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index d8bb00e8c6..a19db74763 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -90,9 +90,12 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->abandoned_heap) + size += (p->hend - p->heap) * sizeof(Eterm); if (p->old_hend && p->old_heap) size += (p->old_hend - p->old_heap) * sizeof(Eterm); + size += p->msg.len * sizeof(ErtsMessage); for (mp = p->msg.first; mp; mp = mp->next) diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8c84303997..ac9e91e31f 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1436,6 +1436,7 @@ void trace_gc(Process *p, Eterm what, Uint size, Eterm msg) { ErtsTracerNif *tnif = NULL; + Eterm* o_hp = NULL; Eterm* hp; Uint sz = 0; Eterm tup; @@ -1446,7 +1447,7 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) if (is_non_value(msg)) { (void) erts_process_gc_info(p, &sz, NULL, 0, 0); - hp = HAlloc(p, sz + 3 + 2); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, (sz + 3 + 2) * sizeof(Eterm)); msg = erts_process_gc_info(p, NULL, &hp, 0, 0); tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; @@ -1455,6 +1456,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, what, msg, THE_NON_VALUE, am_true); + if (o_hp) + erts_free(ERTS_ALC_T_TMP, o_hp); } } diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 7f043e191b..00fa452f79 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -348,7 +348,7 @@ Export *export_list(int i, ErtsCodeIndex code_ix) int export_list_size(ErtsCodeIndex code_ix) { - return export_tables[code_ix].entries; + return erts_index_num_entries(&export_tables[code_ix]); } int export_table_sz(void) diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index c86a2122f6..cd834e2c12 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -91,9 +91,16 @@ index_put_entry(IndexTable* t, void* tmpl) t->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(t->type, sz); t->size += INDEX_PAGE_SIZE; } - t->entries++; p->index = ix; t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; + + /* + * Do a write barrier here to allow readers to do lock free iteration. + * erts_index_num_entries() does matching read barrier. + */ + ERTS_SMP_WRITE_MEMORY_BARRIER; + t->entries++; + return p; } diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index b2e3c0eab5..10f5d1eb39 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -65,6 +65,7 @@ void index_erase_latest_from(IndexTable*, Uint ix); ERTS_GLB_INLINE int index_put(IndexTable*, void*); ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint); +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -78,6 +79,19 @@ erts_index_lookup(IndexTable* t, Uint ix) { return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK]; } + +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t) +{ + int ret = t->entries; + /* + * Do a read barrier here to allow lock free iteration + * on tables where entries are never erased. + * index_put_entry() does matching write barrier. + */ + ERTS_SMP_READ_MEMORY_BARRIER; + return ret; +} + #endif #endif diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7740dd4373..41a7ff4c16 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -859,9 +859,12 @@ int erts_sys_unsetenv(char *key); char *erts_read_env(char *key); void erts_free_read_env(void *value); +#if defined(ERTS_SMP) #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX) extern void sys_thr_resume(erts_tid_t tid); extern void sys_thr_suspend(erts_tid_t tid); +#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2 +#endif #endif /* utils.c */ diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 9c03b3811c..801cef2e56 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -226,11 +226,6 @@ void hipe_handle_exception(Process *c_p) ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ - if (c_p->mbuf) { - erts_printf("%s line %u: p==%p, p->mbuf==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf); - /* erts_garbage_collect(c_p, 0, NULL, 0); */ - } - /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -259,11 +254,6 @@ void hipe_handle_exception(Process *c_p) /* Synthesized to avoid having to generate code for it. */ c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)]; - if (c_p->mbuf) { - /* erts_printf("%s line %u: p==%p, p->mbuf==%p, p->lastbif==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf, c_p->hipe.lastbif); */ - erts_garbage_collect(c_p, 0, NULL, 0); - } - hipe_find_handler(c_p); } diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 2fc802a2c6..e135dbff99 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -119,9 +119,8 @@ erts_smp_atomic_t sys_misc_mem_sz; static void smp_sig_notify(char c); static int sig_notify_fds[2] = {-1, -1}; -#if !defined(ETHR_UNUSABLE_SIGUSRX) && defined(ERTS_THR_HAVE_SIG_FUNCS) +#ifdef ERTS_SYS_SUSPEND_SIGNAL static int sig_suspend_fds[2] = {-1, -1}; -#define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2 #endif #endif diff --git a/erts/emulator/test/port_trace_SUITE_data/echo_drv.c b/erts/emulator/test/port_trace_SUITE_data/echo_drv.c index b545523192..20ec33a594 100644 --- a/erts/emulator/test/port_trace_SUITE_data/echo_drv.c +++ b/erts/emulator/test/port_trace_SUITE_data/echo_drv.c @@ -2,23 +2,30 @@ #include "erl_driver.h" #include <errno.h> #include <string.h> +#include <assert.h> /* ------------------------------------------------------------------------- ** Data types **/ +struct my_thread { + struct my_thread* next; + ErlDrvTid tid; +}; typedef struct _erl_drv_data { ErlDrvPort erlang_port; ErlDrvTermData caller; + struct my_thread* threads; } EchoDrvData; struct remote_send_term { - char *buf; - int len; + struct my_thread thread; ErlDrvTermData port; ErlDrvTermData caller; + int len; + char buf[1]; /* buf[len] */ }; #define ECHO_DRV_NOOP 0 @@ -86,7 +93,7 @@ static ErlDrvEntry echo_drv_entry = { NULL }; -static void send_term_thread(void *); +static void* send_term_thread(void *); /* ------------------------------------------------------------------------- ** Entry functions @@ -111,10 +118,22 @@ static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command) EchoDrvData *echo_drv_data_p = driver_alloc(sizeof(EchoDrvData)); echo_drv_data_p->erlang_port = port; echo_drv_data_p->caller = driver_caller(port); + echo_drv_data_p->threads = NULL; return echo_drv_data_p; } -static void echo_drv_stop(EchoDrvData *data_p) { +static void echo_drv_stop(EchoDrvData *data_p) +{ + struct my_thread* thr = data_p->threads; + + while (thr) { + struct my_thread* next = thr->next; + void* exit_value; + int ret = erl_drv_thread_join(thr->tid, &exit_value); + assert(ret == 0 && exit_value == NULL); + driver_free(thr); + thr = next; + } driver_free(data_p); } @@ -212,14 +231,14 @@ static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) { } case ECHO_DRV_REMOTE_SEND_TERM: { - ErlDrvTid tid; - struct remote_send_term *t = malloc(sizeof(struct remote_send_term)); + struct remote_send_term *t = driver_alloc(sizeof(struct remote_send_term) + len); t->len = len-1; - t->buf = malloc(len-1); t->port = driver_mk_port(port); t->caller = data_p->caller; memcpy(t->buf, buf+1, t->len); - erl_drv_thread_create("tmp_thread", &tid, send_term_thread, t, NULL); + erl_drv_thread_create("tmp_thread", &t->thread.tid, send_term_thread, t, NULL); + t->thread.next = data_p->threads; + data_p->threads = &t->thread; break; } case ECHO_DRV_SAVE_CALLER: @@ -262,7 +281,7 @@ static ErlDrvSSizeT echo_drv_call(ErlDrvData drv_data, return len-command; } -static void send_term_thread(void *a) +static void* send_term_thread(void *a) { struct remote_send_term *t = (struct remote_send_term*)a; ErlDrvTermData term[] = { @@ -273,5 +292,5 @@ static void send_term_thread(void *a) ERL_DRV_TUPLE, 3}; erl_drv_send_term(t->port, t->caller, term, sizeof(term) / sizeof(ErlDrvTermData)); - return; + return NULL; } diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 2289cbabc7..0a6eb7ffac 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -1622,6 +1622,7 @@ spawn_initial_hangarounds(_Cleaner, NP, Max, Len, HAs) when NP > Max -> {Len, HAs}; spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) -> Skip = 30, + wait_for_proc_slots(Skip+3), HA1 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], [{priority, low}]), HA2 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], @@ -1631,6 +1632,15 @@ spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) -> spawn_drop(Skip), spawn_initial_hangarounds(Cleaner, NP+Skip, Max, Len+3, [HA1,HA2,HA3|HAs]). +wait_for_proc_slots(MinFreeSlots) -> + case erlang:system_info(process_limit) - erlang:system_info(process_count) of + FreeSlots when FreeSlots < MinFreeSlots -> + receive after 10 -> ok end, + wait_for_proc_slots(MinFreeSlots); + _FreeSlots -> + ok + end. + spawn_drop(N) when N =< 0 -> ok; spawn_drop(N) -> diff --git a/erts/test/install_SUITE.erl b/erts/test/install_SUITE.erl index 2c7e8972f6..f96dca9563 100644 --- a/erts/test/install_SUITE.erl +++ b/erts/test/install_SUITE.erl @@ -18,7 +18,6 @@ %% %CopyrightEnd% %% - %%%------------------------------------------------------------------- %%% File : install_SUITE.erl %%% Author : Rickard Green @@ -63,12 +62,12 @@ erlang_bindir = "", bindir_symlinks = ""}). -need_symlink_cases() -> +need_symlink_cases() -> [bin_unreachable_absolute, bin_unreachable_relative, bin_same_dir, bin_ok_symlink, bin_dirname_fail, bin_no_use_dirname_fail]. -dont_need_symlink_cases() -> +dont_need_symlink_cases() -> [bin_default, bin_default_dirty, bin_outside_eprfx, bin_outside_eprfx_dirty, bin_not_abs, bin_unreasonable_path, 'bin white space', @@ -78,10 +77,9 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 1}}]. -all() -> +all() -> dont_need_symlink_cases() ++ need_symlink_cases(). - %% %% The test cases %% @@ -533,21 +531,19 @@ bin_no_srcfile(Config) when is_list(Config) -> ChkRes). %% -%% %% Auxiliary functions %% -%% expect(X, X) -> - io:format("result: ~p~n", [X]), + io:format("result: ~tp~n", [X]), io:format("-----------------------------------------------~n", []), ok; expect(X, Y) -> - io:format("expected: ~p~n", [X]), - io:format("got : ~p~n", [Y]), + io:format("expected: ~tp~n", [X]), + io:format("got : ~tp~n", [Y]), io:format("-----------------------------------------------~n", []), ct:fail({X,Y}). - + init_per_suite(Config) -> PD = proplists:get_value(priv_dir, Config), SymLinks = case os:type() of @@ -630,8 +626,8 @@ install_bin(Config, #inst{mkdirs = MkDirs, true -> ok; false -> {comment, "No symlink tests run, since symlinks not working"} end. - - + + install_bin2(Config, Inst, ChkRes) -> install_bin3(Config, Inst#inst{symlinks = false, ln_s = "ln"}, ChkRes), @@ -662,8 +658,6 @@ install_bin2(Config, Inst, ChkRes) -> false -> ok end. - - install_bin3(Config, #inst{cmd_prefix = CMD_PRFX, @@ -690,20 +684,20 @@ install_bin3(Config, ++ "\" --exec-prefix \"" ++ EXEC_PREFIX ++ "\" --test-file \"" ++ ResFile ++ "\" erl erlc", - io:format("CMD_PRFX = \"~s\"~n" - "LN_S = \"~s\"~n" - "BINDIR_SYMLINKS = \"~s\"~n" - "exec_prefix = \"~s\"~n" - "bindir = \"~s\"~n" - "erlang_bindir = \"~s\"~n" - "EXTRA_PREFIX = \"~s\"~n" - "DESTDIR = \"~s\"~n", + io:format("CMD_PRFX = \"~ts\"~n" + "LN_S = \"~ts\"~n" + "BINDIR_SYMLINKS = \"~ts\"~n" + "exec_prefix = \"~ts\"~n" + "bindir = \"~ts\"~n" + "erlang_bindir = \"~ts\"~n" + "EXTRA_PREFIX = \"~ts\"~n" + "DESTDIR = \"~ts\"~n", [CMD_PRFX, LN_S, BINDIR_SYMLINKS, EXEC_PREFIX, BINDIR, ERLANG_BINDIR, EXTRA_PREFIX, DESTDIR]), - io:format("$ ~s~n", [Cmd]), + io:format("$ ~ts~n", [Cmd]), CmdOutput = os:cmd(Cmd), - io:format("~s~n", [CmdOutput]), + io:format("~ts~n", [CmdOutput]), ChkRes(case file:consult(ResFile) of {ok, [Res]} -> Res; Err -> exit({result, Err}) diff --git a/erts/vsn.mk b/erts/vsn.mk index af0be85062..028b114068 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 8.2 +VSN = 8.2.1 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 77588af59b..dfa321c901 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -22,6 +22,7 @@ {vsn, "%VSN%"}, {modules, [ct_cover, ct, + ct_default_gl, ct_event, ct_framework, ct_ftp, diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index b1eddfedd7..2f0fc2e05a 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -70,7 +70,8 @@ MODULES= \ test_server_SUITE \ test_server_test_lib \ ct_release_test_SUITE \ - ct_log_SUITE + ct_log_SUITE \ + ct_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_SUITE.erl b/lib/common_test/test/ct_SUITE.erl new file mode 100644 index 0000000000..eb98c2544f --- /dev/null +++ b/lib/common_test/test/ct_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [app_file, appup_file]. + +%%%----------------------------------------------------------------- +%%% Test cases + +app_file(_Config) -> + ok = test_server:app_test(common_test), + ok. + +appup_file(_Config) -> + ok = test_server:appup_test(common_test). + diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 690d0af1bb..bc716fb5e3 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -70,20 +70,20 @@ suite() -> all() -> all(suite). -all(suite) -> +all(suite) -> lists:reverse( [ one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, faulty_cth_exit_in_init, faulty_cth_exit_in_id, - faulty_cth_exit_in_init_scope_suite, minimal_cth, - minimal_and_maximal_cth, faulty_cth_undef, + faulty_cth_exit_in_init_scope_suite, minimal_cth, + minimal_and_maximal_cth, faulty_cth_undef, scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, - scope_per_suite_state_cth, scope_per_group_state_cth, + scope_per_suite_state_cth, scope_per_group_state_cth, scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, - state_update_cth, options_cth, same_id_cth, + state_update_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, data_dir, cth_log ] @@ -96,10 +96,10 @@ all(suite) -> %%%----------------------------------------------------------------- %%% -one_cth(Config) when is_list(Config) -> +one_cth(Config) when is_list(Config) -> do_test(one_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth], Config). -two_cth(Config) when is_list(Config) -> +two_cth(Config) when is_list(Config) -> do_test(two_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth,empty_cth], Config). @@ -119,13 +119,13 @@ minimal_cth(Config) when is_list(Config) -> minimal_and_maximal_cth(Config) when is_list(Config) -> do_test(minimal_and_maximal_cth, "ct_cth_empty_SUITE.erl", [minimal_cth, empty_cth],Config). - + faulty_cth_undef(Config) when is_list(Config) -> do_test(faulty_cth_undef, "ct_cth_empty_SUITE.erl", [undef_cth],Config). faulty_cth_exit_in_init_scope_suite(Config) when is_list(Config) -> - do_test(faulty_cth_exit_in_init_scope_suite, + do_test(faulty_cth_exit_in_init_scope_suite, "ct_exit_in_init_scope_suite_cth_SUITE.erl", [],Config). @@ -205,7 +205,7 @@ state_update_cth(Config) when is_list(Config) -> options_cth(Config) when is_list(Config) -> do_test(options_cth, "ct_cth_empty_SUITE.erl", [{empty_cth,[test]}],Config). - + same_id_cth(Config) when is_list(Config) -> do_test(same_id_cth, "ct_cth_empty_SUITE.erl", [same_id_cth,same_id_cth],Config). @@ -227,9 +227,10 @@ data_dir(Config) when is_list(Config) -> do_test(data_dir, "ct_data_dir_SUITE.erl", [verify_data_dir_cth],Config). -cth_log(Config) when is_list(Config) -> +cth_log(Config) when is_list(Config) -> %% test that cth_log_redirect writes properly to %% unexpected I/O log + ct:timetrap({minutes,10}), StartOpts = do_test(cth_log, "cth_log_SUITE.erl", [], Config), Logdir = proplists:get_value(logdir, StartOpts), UnexpIoLogs = @@ -266,7 +267,6 @@ do_test(Tag, SWC, CTHs, Config, Res) -> do_test(Tag, SWC, CTHs, Config, Res, 2). do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> - DataDir = ?config(data_dir, Config), Suites = filelib:wildcard( filename:join([DataDir,"cth/tests",SuiteWildCard])), @@ -275,7 +275,7 @@ do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> Res = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), - ct_test_support:log_events(Tag, + ct_test_support:log_events(Tag, reformat(Events, ?eh), ?config(priv_dir, Config), Opts), @@ -328,7 +328,7 @@ test_events(one_empty_cth) -> {?eh,cth,{empty_cth,pre_end_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, [ct_cth_empty_SUITE,'$proplist',[]]}}, @@ -360,7 +360,7 @@ test_events(two_empty_cth) -> {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, @@ -402,7 +402,7 @@ test_events(minimal_cth) -> {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -426,7 +426,7 @@ test_events(minimal_and_maximal_cth) -> {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, @@ -452,11 +452,11 @@ test_events(faulty_cth_undef) -> {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed, FailReason}}}, {?eh,cth,{'_',on_tc_skip,'_'}}, - + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,end_per_suite, {failed, FailReason}}}, {?eh,cth,{'_',on_tc_skip,'_'}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; @@ -515,7 +515,7 @@ test_events(scope_per_suite_cth) -> {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [ct_scope_per_suite_cth_SUITE,'$proplist',[]]}}, @@ -541,7 +541,7 @@ test_events(scope_suite_cth) -> {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist','_',[]]}}, @@ -563,18 +563,18 @@ test_events(scope_per_group_cth) -> {?eh,cth,{'_',init,['_',[]]}}, {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, {?eh,cth,{'_',terminate,[[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}], - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -595,7 +595,7 @@ test_events(scope_per_suite_state_cth) -> {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [ct_scope_per_suite_state_cth_SUITE,'$proplist',[test]]}}, @@ -621,7 +621,7 @@ test_events(scope_suite_state_cth) -> {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist',[test]]}}, {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist','_',[test]]}}, @@ -643,18 +643,18 @@ test_events(scope_per_group_state_cth) -> {?eh,cth,{'_',init,['_',[test]]}}, {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}}, {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}}, {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}}, {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}}, {?eh,cth,{'_',terminate,[[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}], - + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,end_per_suite}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -666,7 +666,7 @@ test_events(fail_pre_suite_cth) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{'_',init,['_',[]]}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist', @@ -676,7 +676,7 @@ test_events(fail_pre_suite_cth) -> {?eh,cth,{'_',on_tc_fail, [init_per_suite,{failed,"Test failure"},[]]}}, - + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed,{ct_cth_empty_SUITE,init_per_suite, {failed,"Test failure"}}}}}, @@ -685,7 +685,7 @@ test_events(fail_pre_suite_cth) -> {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}},[]]}}, - + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}}}, @@ -694,7 +694,7 @@ test_events(fail_pre_suite_cth) -> {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}},[]]}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, {?eh,stop_logging,[]} @@ -733,7 +733,7 @@ test_events(fail_post_suite_cth) -> {failed,{ct_cth_empty_SUITE,init_per_suite, {failed,"Test failure"}}}}}, {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}}, - + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}}}, @@ -758,7 +758,7 @@ test_events(skip_pre_suite_cth) -> {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -772,18 +772,18 @@ test_events(skip_pre_end_cth) -> {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,init_per_suite}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,init_per_suite,ok}}, - + [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, @@ -808,7 +808,7 @@ test_events(skip_post_suite_cth) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{'_',init,['_',[]]}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, @@ -818,9 +818,9 @@ test_events(skip_post_suite_cth) -> {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, {?eh,stop_logging,[]} @@ -844,7 +844,7 @@ test_events(recover_post_suite_cth) -> {?eh,cth,{'_',post_end_per_testcase, [test_case, contains([tc_status]),'_',[]]}}, {?eh,tc_done,{Suite,test_case,ok}}, - + {?eh,tc_start,{Suite,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [Suite,not_contains([tc_status]),[]]}}, @@ -861,7 +861,7 @@ test_events(update_config_cth) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{'_',init,['_',[]]}}, - + {?eh,tc_start,{ct_update_config_SUITE,init_per_suite}}, {?eh,cth,{'_',pre_init_per_suite, [ct_update_config_SUITE,contains([]),[]]}}, @@ -941,7 +941,7 @@ test_events(update_config_cth) -> pre_init_per_suite]), ok,[]]}}, {?eh,tc_done,{ct_update_config_SUITE,{end_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_update_config_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [ct_update_config_SUITE,contains( @@ -974,7 +974,7 @@ test_events(state_update_cth) -> {?eh,cth,{'_',init,['_',[]]}}, {?eh,cth,{'_',init,['_',[]]}}, {?eh,tc_start,{'_',init_per_suite}}, - + {?eh,tc_done,{'_',end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[contains( @@ -1021,7 +1021,7 @@ test_events(options_cth) -> {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}}, {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, [ct_cth_empty_SUITE,'$proplist',[test]]}}, @@ -1058,7 +1058,7 @@ test_events(same_id_cth) -> {negative, {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {negative, @@ -1115,17 +1115,14 @@ test_events(fail_n_skip_with_minimal_cth) -> ]; test_events(prio_cth) -> - GenPre = fun(Func,States) -> - [{?eh,cth,{'_',Func,['_','_',State]}} || - State <- States] + [{?eh,cth,{'_',Func,['_','_',State]}} || State <- States] end, GenPost = fun(Func,States) -> - [{?eh,cth,{'_',Func,['_','_','_',State]}} || - State <- States] + [{?eh,cth,{'_',Func,['_','_','_',State]}} || State <- States] end, - + [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}] ++ @@ -1136,7 +1133,7 @@ test_events(prio_cth) -> [[1100,100],[600,200],[600,600],[700],[800],[900],[1000], [1200,1050],[1100],[1200]]) ++ [{?eh,tc_done,{ct_cth_prio_SUITE,init_per_suite,ok}}, - + [{?eh,tc_start,{ct_cth_prio_SUITE,{init_per_group,'_',[]}}}] ++ GenPre(pre_init_per_group, @@ -1147,7 +1144,7 @@ test_events(prio_cth) -> [900],[900,900],[500,900],[1000],[1200,1050], [1100],[1200]]) ++ [{?eh,tc_done,{ct_cth_prio_SUITE,{init_per_group,'_',[]},ok}}] ++ - + [{?eh,tc_start,{ct_cth_prio_SUITE,test_case}}] ++ GenPre(pre_init_per_testcase, [[1100,100],[600,200],[600,600],[600],[700],[800], @@ -1161,7 +1158,7 @@ test_events(prio_cth) -> [{?eh,tc_done,{ct_cth_prio_SUITE,test_case,ok}}, {?eh,tc_start,{ct_cth_prio_SUITE,{end_per_group,'_',[]}}}] ++ - GenPre(pre_end_per_group, + GenPre(pre_end_per_group, lists:reverse( [[1100,100],[600,200],[600,600],[600],[700],[800], [900],[900,900],[500,900],[1000],[1200,1050], @@ -1300,7 +1297,7 @@ test_events(cth_log) -> [{suite,cth_log_SUITE},parallel]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g1, [{suite,cth_log_SUITE},parallel]},ok}}]}, - + {?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -1309,7 +1306,6 @@ test_events(cth_log) -> test_events(ok) -> ok. - %% test events help functions contains(List) -> fun(Proplist) when is_list(Proplist) -> diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 434360d294..e37ca31704 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -173,17 +173,25 @@ env_default_opts() -> do_compile(Input, Opts0) -> Opts = expand_opts(Opts0), - {Pid,Ref} = - spawn_monitor(fun() -> - exit(try - internal(Input, Opts) - catch - error:Reason -> - {error,Reason} - end) - end), - receive - {'DOWN',Ref,process,Pid,Rep} -> Rep + IntFun = fun() -> try + internal(Input, Opts) + catch + error:Reason -> + {error,Reason} + end + end, + %% Dialyzer has already spawned workers. + case lists:member(dialyzer, Opts) of + true -> + IntFun(); + false -> + {Pid,Ref} = + spawn_monitor(fun() -> + exit(IntFun()) + end), + receive + {'DOWN',Ref,process,Pid,Rep} -> Rep + end end. expand_opts(Opts0) -> diff --git a/lib/compiler/test/lc_SUITE.erl b/lib/compiler/test/lc_SUITE.erl index 3cb49433ce..adb96fb87d 100644 --- a/lib/compiler/test/lc_SUITE.erl +++ b/lib/compiler/test/lc_SUITE.erl @@ -19,7 +19,7 @@ %% -module(lc_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0, groups/0, init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, basic/1,deeply_nested/1,no_generator/1, @@ -32,11 +32,11 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,1}}]. -all() -> +all() -> test_lib:recompile(?MODULE), [{group,p}]. -groups() -> +groups() -> [{p,test_lib:parallel(), [basic, deeply_nested, @@ -214,6 +214,7 @@ shadow(Config) when is_list(Config) -> ok. effect(Config) when is_list(Config) -> + ct:timetrap({minutes,10}), [{42,{a,b,c}}] = do_effect(fun(F, L) -> [F({V1,V2}) || @@ -240,7 +241,7 @@ do_effect(Lc, L) -> lists:reverse(erase(?MODULE)). id(I) -> I. - + fc(Args, {'EXIT',{function_clause,[{?MODULE,_,Args,_}|_]}}) -> ok; fc(Args, {'EXIT',{function_clause,[{?MODULE,_,Arity,_}|_]}}) when length(Args) =:= Arity -> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index da8626e38a..a87b480f60 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -277,10 +277,11 @@ hmac_final_n(Context, HashLen) -> des3_cbc | des3_cbf | des3_cfb | des_ede3 | blowfish_cbc | blowfish_cfb64 | blowfish_ofb64 | aes_cbc128 | aes_cfb8 | aes_cfb128 | aes_cbc256 | aes_ige256 | - aes_cbc | + aes_cbc | rc2_cbc, - Key::iodata(), Ivec::binary(), Data::iodata()) -> binary(); - (aes_gcm | chacha20_poly1305, Key::iodata(), Ivec::binary(), {AAD::binary(), Data::iodata()}) -> {binary(), binary()}. + Key::iodata(), Ivec::binary(), Data::iodata()) -> binary(); + (aes_gcm | chacha20_poly1305, Key::iodata(), Ivec::binary(), {AAD::binary(), Data::iodata()}) -> {binary(), binary()}; + (aes_gcm, Key::iodata(), Ivec::binary(), {AAD::binary(), Data::iodata(), TagLength::1..16}) -> {binary(), binary()}. block_encrypt(Type, Key, Ivec, Data) when Type =:= des_cbc; Type =:= des_cfb; diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl index 08e55a78bd..4e18058993 100644 --- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl @@ -101,9 +101,9 @@ loop(#server_state{parent = Parent} = State, {AnalPid, cserver, CServer, Plt} -> send_codeserver_plt(Parent, CServer, Plt), loop(State, Analysis, ExtCalls); - {AnalPid, done, Plt, DocPlt} -> + {AnalPid, done, MiniPlt, DocPlt} -> send_ext_calls(Parent, ExtCalls), - send_analysis_done(Parent, Plt, DocPlt); + send_analysis_done(Parent, MiniPlt, DocPlt); {AnalPid, ext_calls, NewExtCalls} -> loop(State, Analysis, NewExtCalls); {AnalPid, ext_types, ExtTypes} -> @@ -121,6 +121,7 @@ loop(#server_state{parent = Parent} = State, %% The Analysis %%-------------------------------------------------------------------- +%% Calls to erlang:garbage_collect() help to reduce the heap size. analysis_start(Parent, Analysis, LegalWarnings) -> CServer = dialyzer_codeserver:new(), Plt = Analysis#analysis.plt, @@ -157,12 +158,9 @@ analysis_start(Parent, Analysis, LegalWarnings) -> TmpCServer1 = dialyzer_codeserver:set_temp_records(MergedRecords, TmpCServer0), TmpCServer2 = dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), + erlang:garbage_collect(), ?timing(State#analysis_state.timing_server, "remote", - begin - TmpCServer3 = - dialyzer_utils:process_record_remote_types(TmpCServer2), - dialyzer_contracts:process_contract_remote_types(TmpCServer3) - end) + contracts_and_records(TmpCServer2)) catch throw:{error, _ErrorMsg} = Error -> exit(Error) end, @@ -171,48 +169,75 @@ analysis_start(Parent, Analysis, LegalWarnings) -> NewPlt1 = dialyzer_plt:insert_exported_types(NewPlt0, ExpTypes), State0 = State#analysis_state{plt = NewPlt1}, dump_callgraph(Callgraph, State0, Analysis), - State1 = State0#analysis_state{codeserver = NewCServer}, %% Remove all old versions of the files being analyzed AllNodes = dialyzer_callgraph:all_nodes(Callgraph), - Plt1 = dialyzer_plt:delete_list(NewPlt1, AllNodes), + Plt1_a = dialyzer_plt:delete_list(NewPlt1, AllNodes), + Plt1 = dialyzer_plt:insert_callbacks(Plt1_a, NewCServer), + State1 = State0#analysis_state{codeserver = NewCServer, plt = Plt1}, Exports = dialyzer_codeserver:get_exports(NewCServer), + NonExports = sets:subtract(sets:from_list(AllNodes), Exports), + NonExportsList = sets:to_list(NonExports), NewCallgraph = case Analysis#analysis.race_detection of true -> dialyzer_callgraph:put_race_detection(true, Callgraph); false -> Callgraph end, - State2 = analyze_callgraph(NewCallgraph, State1#analysis_state{plt = Plt1}), + State2 = analyze_callgraph(NewCallgraph, State1), + #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2, dialyzer_callgraph:dispose_race_server(NewCallgraph), rcv_and_send_ext_types(Parent), - NonExports = sets:subtract(sets:from_list(AllNodes), Exports), - NonExportsList = sets:to_list(NonExports), - Plt2 = dialyzer_plt:delete_list(State2#analysis_state.plt, NonExportsList), - send_codeserver_plt(Parent, CServer, State2#analysis_state.plt), - send_analysis_done(Parent, Plt2, State2#analysis_state.doc_plt). + %% Since the PLT is never used, a dummy is sent: + DummyPlt = dialyzer_plt:new(), + send_codeserver_plt(Parent, CServer, DummyPlt), + MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList), + send_analysis_done(Parent, MiniPlt3, DocPlt). + +contracts_and_records(CodeServer) -> + Fun = contrs_and_recs(CodeServer), + {Pid, Ref} = erlang:spawn_monitor(Fun), + dialyzer_codeserver:give_away(CodeServer, Pid), + Pid ! {self(), go}, + receive {'DOWN', Ref, process, Pid, Return} -> + Return + end. + +-spec contrs_and_recs(dialyzer_codeserver:codeserver()) -> + fun(() -> no_return()). + +contrs_and_recs(TmpCServer2) -> + fun() -> + Parent = receive {Pid, go} -> Pid end, + {TmpCServer3, RecordDict} = + dialyzer_utils:process_record_remote_types(TmpCServer2), + TmpServer4 = + dialyzer_contracts:process_contract_remote_types(TmpCServer3, + RecordDict), + dialyzer_codeserver:give_away(TmpServer4, Parent), + exit(TmpServer4) + end. analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver, doc_plt = DocPlt, + plt = Plt, timing_server = TimingServer, parent = Parent, solvers = Solvers} = State) -> - Plt = dialyzer_plt:insert_callbacks(State#analysis_state.plt, Codeserver), - {NewPlt, NewDocPlt} = - case State#analysis_state.analysis_type of - plt_build -> - NewPlt0 = - dialyzer_succ_typings:analyze_callgraph(Callgraph, Plt, Codeserver, - TimingServer, Solvers, Parent), - {NewPlt0, DocPlt}; - succ_typings -> - {Warnings, NewPlt0, NewDocPlt0} = - dialyzer_succ_typings:get_warnings(Callgraph, Plt, DocPlt, Codeserver, - TimingServer, Solvers, Parent), - Warnings1 = filter_warnings(Warnings, Codeserver), - send_warnings(State#analysis_state.parent, Warnings1), - {NewPlt0, NewDocPlt0} - end, - dialyzer_callgraph:delete(Callgraph), - State#analysis_state{plt = NewPlt, doc_plt = NewDocPlt}. + case State#analysis_state.analysis_type of + plt_build -> + NewMiniPlt = + dialyzer_succ_typings:analyze_callgraph(Callgraph, Plt, Codeserver, + TimingServer, Solvers, Parent), + dialyzer_callgraph:delete(Callgraph), + State#analysis_state{plt = NewMiniPlt, doc_plt = DocPlt}; + succ_typings -> + {Warnings, NewMiniPlt, NewDocPlt} = + dialyzer_succ_typings:get_warnings(Callgraph, Plt, DocPlt, Codeserver, + TimingServer, Solvers, Parent), + dialyzer_callgraph:delete(Callgraph), + Warnings1 = filter_warnings(Warnings, Codeserver), + send_warnings(State#analysis_state.parent, Warnings1), + State#analysis_state{plt = NewMiniPlt, doc_plt = NewDocPlt} + end. %%-------------------------------------------------------------------- %% Build the callgraph and fill the codeserver. @@ -569,8 +594,9 @@ is_ok_fun({_Filename, _Line, {_M, _F, _A} = MFA}, Codeserver) -> is_ok_tag(Tag, {_F, _L, MorMFA}, Codeserver) -> not dialyzer_utils:is_suppressed_tag(MorMFA, Tag, Codeserver). -send_analysis_done(Parent, Plt, DocPlt) -> - Parent ! {self(), done, Plt, DocPlt}, +send_analysis_done(Parent, MiniPlt, DocPlt) -> + ok = dialyzer_plt:give_away(MiniPlt, Parent), + Parent ! {self(), done, MiniPlt, DocPlt}, ok. send_ext_calls(_Parent, none) -> @@ -583,7 +609,7 @@ send_ext_types(Parent, ExtTypes) -> Parent ! {self(), ext_types, ExtTypes}, ok. -send_codeserver_plt(Parent, CServer, Plt ) -> +send_codeserver_plt(Parent, CServer, Plt) -> Parent ! {self(), cserver, CServer, Plt}, ok. @@ -602,14 +628,14 @@ format_bad_calls([{{_, _, _}, {_, module_info, A}}|Left], CodeServer, Acc) format_bad_calls([{FromMFA, {M, F, A} = To}|Left], CodeServer, Acc) -> {_Var, FunCode} = dialyzer_codeserver:lookup_mfa_code(FromMFA, CodeServer), Msg = {call_to_missing, [M, F, A]}, - {File, Line} = find_call_file_and_line(FunCode, To), + {File, Line} = find_call_file_and_line(FromMFA, FunCode, To, CodeServer), WarningInfo = {File, Line, FromMFA}, NewAcc = [{?WARN_CALLGRAPH, WarningInfo, Msg}|Acc], format_bad_calls(Left, CodeServer, NewAcc); format_bad_calls([], _CodeServer, Acc) -> Acc. -find_call_file_and_line(Tree, MFA) -> +find_call_file_and_line({Module, _, _}, Tree, MFA, CodeServer) -> Fun = fun(SubTree, Acc) -> case cerl:is_c_call(SubTree) of @@ -622,7 +648,7 @@ find_call_file_and_line(Tree, MFA) -> case {cerl:concrete(M), cerl:concrete(F), A} of MFA -> Ann = cerl:get_ann(SubTree), - [{get_file(Ann), get_line(Ann)}|Acc]; + [{get_file(CodeServer, Module, Ann), get_line(Ann)}|Acc]; {erlang, make_fun, 3} -> [CA1, CA2, CA3] = cerl:call_args(SubTree), case @@ -638,7 +664,8 @@ find_call_file_and_line(Tree, MFA) -> of MFA -> Ann = cerl:get_ann(SubTree), - [{get_file(Ann), get_line(Ann)}|Acc]; + [{get_file(CodeServer, Module, Ann), + get_line(Ann)}|Acc]; _ -> Acc end; @@ -658,8 +685,10 @@ get_line([Line|_]) when is_integer(Line) -> Line; get_line([_|Tail]) -> get_line(Tail); get_line([]) -> -1. -get_file([{file, File}|_]) -> File; -get_file([_|Tail]) -> get_file(Tail). +get_file(Codeserver, Module, [{file, FakeFile}|_]) -> + dialyzer_codeserver:translate_fake_file(Codeserver, Module, FakeFile); +get_file(Codeserver, Module, [_|Tail]) -> + get_file(Codeserver, Module, Tail). -spec dump_callgraph(dialyzer_callgraph:callgraph(), #analysis_state{}, #analysis{}) -> 'ok'. diff --git a/lib/dialyzer/src/dialyzer_behaviours.erl b/lib/dialyzer/src/dialyzer_behaviours.erl index 5623929a43..524ae047e2 100644 --- a/lib/dialyzer/src/dialyzer_behaviours.erl +++ b/lib/dialyzer/src/dialyzer_behaviours.erl @@ -62,9 +62,9 @@ check_callbacks(Module, Attrs, Records, Plt, Codeserver) -> _ -> MFA = {Module,module_info,0}, {_Var,Code} = dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver), - File = get_file(cerl:get_ann(Code)), + File = get_file(Codeserver, Module, cerl:get_ann(Code)), State = #state{plt = Plt, filename = File, behlines = BehLines, - codeserver = Codeserver, records = Records}, + codeserver = Codeserver, records = Records}, Warnings = get_warnings(Module, Behaviours, State), [add_tag_warning_info(Module, W, State) || W <- Warnings] end. @@ -213,12 +213,15 @@ add_tag_warning_info(Module, {_Tag, [_B, Fun, Arity|_R]} = Warn, State) -> dialyzer_codeserver:lookup_mfa_code({Module, Fun, Arity}, State#state.codeserver), Anns = cerl:get_ann(FunCode), - WarningInfo = {get_file(Anns), get_line(Anns), {Module, Fun, Arity}}, + File = get_file(State#state.codeserver, Module, Anns), + WarningInfo = {File, get_line(Anns), {Module, Fun, Arity}}, {?WARN_BEHAVIOUR, WarningInfo, Warn}. get_line([Line|_]) when is_integer(Line) -> Line; get_line([_|Tail]) -> get_line(Tail); get_line([]) -> -1. -get_file([{file, File}|_]) -> File; -get_file([_|Tail]) -> get_file(Tail). +get_file(Codeserver, Module, [{file, FakeFile}|_]) -> + dialyzer_codeserver:translate_fake_file(Codeserver, Module, FakeFile); +get_file(Codeserver, Module, [_|Tail]) -> + get_file(Codeserver, Module, Tail). diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl index 50abb22009..5e02e7a2cc 100644 --- a/lib/dialyzer/src/dialyzer_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_callgraph.erl @@ -119,7 +119,11 @@ -opaque callgraph() :: #callgraph{}. --type active_digraph() :: {'d', digraph:graph()} | {'e', ets:tid(), ets:tid()}. +-type active_digraph() :: {'d', digraph:graph()} + | {'e', + Out :: ets:tid(), + In :: ets:tid(), + Map :: ets:tid()}. %%---------------------------------------------------------------------- @@ -248,24 +252,30 @@ find_non_local_calls([], Set) -> -spec get_depends_on(scc() | module(), callgraph()) -> [scc()]. -get_depends_on(SCC, #callgraph{active_digraph = {'e', Out, _In}}) -> - case ets_lookup_dict(SCC, Out) of - {ok, Value} -> Value; - error -> [] - end; +get_depends_on(SCC, #callgraph{active_digraph = {'e', Out, _In, Maps}}) -> + lookup_scc(SCC, Out, Maps); get_depends_on(SCC, #callgraph{active_digraph = {'d', DG}}) -> digraph:out_neighbours(DG, SCC). -spec get_required_by(scc() | module(), callgraph()) -> [scc()]. -get_required_by(SCC, #callgraph{active_digraph = {'e', _Out, In}}) -> - case ets_lookup_dict(SCC, In) of - {ok, Value} -> Value; - error -> [] - end; +get_required_by(SCC, #callgraph{active_digraph = {'e', _Out, In, Maps}}) -> + lookup_scc(SCC, In, Maps); get_required_by(SCC, #callgraph{active_digraph = {'d', DG}}) -> digraph:in_neighbours(DG, SCC). +lookup_scc(SCC, Table, Maps) -> + case ets_lookup_dict({'scc', SCC}, Maps) of + {ok, SCCInt} -> + case ets_lookup_dict(SCCInt, Table) of + {ok, Ints} -> + [ets:lookup_element(Maps, Int, 2) || Int <- Ints]; + error -> + [] + end; + error -> [] + end. + %%---------------------------------------------------------------------- %% Handling of modules & SCCs %%---------------------------------------------------------------------- @@ -582,9 +592,10 @@ digraph_delete(DG) -> active_digraph_delete({'d', DG}) -> digraph:delete(DG); -active_digraph_delete({'e', Out, In}) -> +active_digraph_delete({'e', Out, In, Maps}) -> ets:delete(Out), - ets:delete(In). + ets:delete(In), + ets:delete(Maps). digraph_edges(DG) -> digraph:edges(DG). @@ -758,37 +769,28 @@ to_ps(#callgraph{} = CG, File, Args) -> ok. condensation(G) -> - SCs = digraph_utils:strong_components(G), - V2I = ets:new(condensation_v2i, []), - I2C = ets:new(condensation_i2c, []), - I2I = ets:new(condensation_i2i, [bag]), - CFun = - fun(SC, N) -> - lists:foreach(fun(V) -> true = ets:insert(V2I, {V,N}) end, SC), - true = ets:insert(I2C, {N, SC}), - N + 1 - end, - lists:foldl(CFun, 1, SCs), - Fun1 = - fun({V1, V2}) -> - I1 = ets:lookup_element(V2I, V1, 2), - I2 = ets:lookup_element(V2I, V2, 2), - I1 =:= I2 orelse ets:insert(I2I, {I1, I2}) - end, - lists:foreach(Fun1, digraph:edges(G)), - Fun3 = - fun({I1, I2}, {Out, In}) -> - SC1 = ets:lookup_element(I2C, I1, 2), - SC2 = ets:lookup_element(I2C, I2, 2), - {dict:append(SC1, SC2, Out), dict:append(SC2, SC1, In)} - end, - {OutDict, InDict} = ets:foldl(Fun3, {dict:new(), dict:new()}, I2I), - [OutETS, InETS] = + SCCs = digraph_utils:strong_components(G), + %% Assign unique numbers to SCCs: + Ints = lists:seq(1, length(SCCs)), + IntToSCC = lists:zip(Ints, SCCs), + IntScc = sofs:relation(IntToSCC, [{int, scc}]), + %% Subsitute strong components for vertices in edges using the + %% unique numbers: + C2V = sofs:relation([{SC, V} || SC <- SCCs, V <- SC], [{scc, v}]), + I2V = sofs:relative_product(IntScc, C2V), % [{v, int}] + Es = sofs:relation(digraph:edges(G), [{v, v}]), + R1 = sofs:relative_product(I2V, Es), + R2 = sofs:relative_product(I2V, sofs:converse(R1)), + %% Create in- and out-neighbours: + In = sofs:relation_to_family(sofs:strict_relation(R2)), + R3 = sofs:converse(R2), + Out = sofs:relation_to_family(sofs:strict_relation(R3)), + [OutETS, InETS, MapsETS] = [ets:new(Name,[{read_concurrency, true}]) || - Name <- [callgraph_deps_out, callgraph_deps_in]], - ets:insert(OutETS, dict:to_list(OutDict)), - ets:insert(InETS, dict:to_list(InDict)), - ets:delete(V2I), - ets:delete(I2C), - ets:delete(I2I), - {{'e', OutETS, InETS}, SCs}. + Name <- [callgraph_deps_out, callgraph_deps_in, callgraph_scc_map]], + ets:insert(OutETS, sofs:to_external(Out)), + ets:insert(InETS, sofs:to_external(In)), + %% Create mappings from SCCs to unique integers, and the inverse: + ets:insert(MapsETS, lists:zip([{'scc', SCC} || SCC<- SCCs], Ints)), + ets:insert(MapsETS, IntToSCC), + {{'e', OutETS, InETS, MapsETS}, SCCs}. diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index fc56693ea3..e8c1613a33 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -637,8 +637,8 @@ cl_loop(State, LogCache) -> {BackendPid, warnings, Warnings} -> NewState = store_warnings(State, Warnings), cl_loop(NewState, LogCache); - {BackendPid, done, NewPlt, _NewDocPlt} -> - return_value(State, NewPlt); + {BackendPid, done, NewMiniPlt, _NewDocPlt} -> + return_value(State, NewMiniPlt); {BackendPid, ext_calls, ExtCalls} -> cl_loop(State#cl_state{external_calls = ExtCalls}, LogCache); {BackendPid, ext_types, ExtTypes} -> @@ -654,6 +654,7 @@ cl_loop(State, LogCache) -> cl_error(State, Msg); _Other -> %% io:format("Received ~p\n", [_Other]), + %% Note: {BackendPid, cserver, CodeServer, Plt} is ignored. cl_loop(State, LogCache) end. @@ -699,10 +700,13 @@ return_value(State = #cl_state{erlang_mode = ErlangMode, output_plt = OutputPlt, plt_info = PltInfo, stored_warnings = StoredWarnings}, - Plt) -> + MiniPlt) -> case OutputPlt =:= none of - true -> ok; - false -> dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo) + true -> + dialyzer_plt:delete(MiniPlt); + false -> + Plt = dialyzer_plt:restore_full_plt(MiniPlt), + dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo) end, UnknownWarnings = unknown_warnings(State), RetValue = diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index 03cd9671af..a5bb4e209c 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -29,7 +29,9 @@ -module(dialyzer_codeserver). -export([delete/1, - finalize_contracts/3, + store_temp_contracts/4, + give_away/2, + finalize_contracts/1, finalize_exported_types/2, finalize_records/2, get_contracts/1, @@ -38,7 +40,9 @@ get_exports/1, get_records/1, get_next_core_label/1, - get_temp_contracts/1, + get_temp_contracts/2, + contracts_modules/1, + store_contracts/4, get_temp_exported_types/1, get_temp_records/1, insert/3, @@ -48,6 +52,7 @@ is_exported/2, lookup_mod_code/2, lookup_mfa_code/2, + lookup_mfa_var_label/2, lookup_mod_records/2, lookup_mod_contracts/2, lookup_mfa_contract/2, @@ -56,21 +61,22 @@ set_next_core_label/2, set_temp_records/2, store_temp_records/3, - store_temp_contracts/4]). + translate_fake_file/3]). --export_type([codeserver/0, fun_meta_info/0]). +-export_type([codeserver/0, fun_meta_info/0, contracts/0]). -include("dialyzer.hrl"). %%-------------------------------------------------------------------- -type dict_ets() :: ets:tid(). +-type map_ets() :: ets:tid(). -type set_ets() :: ets:tid(). -type types() :: erl_types:type_table(). --type mod_records() :: dict:dict(module(), types()). +-type mod_records() :: erl_types:mod_records(). --type contracts() :: dict:dict(mfa(),dialyzer_contracts:file_contract()). +-type contracts() :: #{mfa() => dialyzer_contracts:file_contract()}. -type mod_contracts() :: dict:dict(module(), contracts()). %% A property-list of data compiled from -compile and -dialyzer attributes. @@ -81,16 +87,16 @@ -record(codeserver, {next_core_label = 0 :: label(), code :: dict_ets(), - exported_types :: set_ets() | 'undefined', % set(mfa()) - records :: dict_ets() | 'undefined', - contracts :: dict_ets() | 'undefined', - callbacks :: dict_ets() | 'undefined', + exported_types :: set_ets(), % set(mfa()) + records :: map_ets(), + contracts :: map_ets(), + callbacks :: map_ets(), fun_meta_info :: dict_ets(), % {mfa(), meta_info()} exports :: 'clean' | set_ets(), % set(mfa()) temp_exported_types :: 'clean' | set_ets(), % set(mfa()) - temp_records :: 'clean' | dict_ets(), - temp_contracts :: 'clean' | dict_ets(), - temp_callbacks :: 'clean' | dict_ets() + temp_records :: 'clean' | map_ets(), + temp_contracts :: 'clean' | map_ets(), + temp_callbacks :: 'clean' | map_ets() }). -opaque codeserver() :: #codeserver{}. @@ -104,7 +110,7 @@ ets_dict_find(Key, Table) -> _:_ -> error end. -ets_dict_store(Key, Element, Table) -> +ets_map_store(Key, Element, Table) -> true = ets:insert(Table, {Key, Element}), Table. @@ -128,9 +134,6 @@ ets_set_to_set(Table) -> Fold = fun({E}, Set) -> sets:add_element(E, Set) end, ets:foldl(Fold, sets:new(), Table). -ets_read_concurrent_table(Name) -> - ets:new(Name, [{read_concurrency, true}]). - %%-------------------------------------------------------------------- -spec new() -> codeserver(). @@ -138,6 +141,13 @@ ets_read_concurrent_table(Name) -> new() -> CodeOptions = [compressed, public, {read_concurrency, true}], Code = ets:new(dialyzer_codeserver_code, CodeOptions), + ReadOptions = [compressed, {read_concurrency, true}], + [Contracts, Callbacks, Records, ExportedTypes] = + [ets:new(Name, ReadOptions) || + Name <- [dialyzer_codeserver_contracts, + dialyzer_codeserver_callbacks, + dialyzer_codeserver_records, + dialyzer_codeserver_exported_types]], TempOptions = [public, {write_concurrency, true}], [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts, TempCallbacks] = @@ -150,6 +160,10 @@ new() -> #codeserver{code = Code, exports = Exports, fun_meta_info = FunMetaInfo, + exported_types = ExportedTypes, + records = Records, + contracts = Contracts, + callbacks = Callbacks, temp_exported_types = TempExportedTypes, temp_records = TempRecords, temp_contracts = TempContracts, @@ -170,13 +184,15 @@ insert(Mod, ModCode, CS) -> Exports = cerl:module_exports(ModCode), Attrs = cerl:module_attrs(ModCode), Defs = cerl:module_defs(ModCode), + {Files, SmallDefs} = compress_file_anno(Defs), As = cerl:get_ann(ModCode), Funs = [{{Mod, cerl:fname_id(Var), cerl:fname_arity(Var)}, - Val} || Val = {Var, _Fun} <- Defs], - Keys = [Key || {Key, _Value} <- Funs], + Val, {Var, cerl_trees:get_label(Fun)}} || Val = {Var, Fun} <- SmallDefs], + Keys = [Key || {Key, _Value, _Label} <- Funs], ModEntry = {Mod, {Name, Exports, Attrs, Keys, As}}, - true = ets:insert(CS#codeserver.code, [ModEntry|Funs]), + ModFileEntry = {{mod, Mod}, Files}, + true = ets:insert(CS#codeserver.code, [ModEntry, ModFileEntry|Funs]), CS. -spec get_temp_exported_types(codeserver()) -> sets:set(mfa()). @@ -220,12 +236,12 @@ get_exports(#codeserver{exports = Exports}) -> -spec finalize_exported_types(sets:set(mfa()), codeserver()) -> codeserver(). -finalize_exported_types(Set, CS) -> - ExportedTypes = ets_read_concurrent_table(dialyzer_codeserver_exported_types), +finalize_exported_types(Set, + #codeserver{exported_types = ExportedTypes, + temp_exported_types = TempETypes} = CS) -> true = ets_set_insert_set(Set, ExportedTypes), - TempExpTypes = CS#codeserver.temp_exported_types, - true = ets:delete(TempExpTypes), - CS#codeserver{exported_types = ExportedTypes, temp_exported_types = clean}. + true = ets:delete(TempETypes), + CS#codeserver{temp_exported_types = clean}. -spec lookup_mod_code(atom(), codeserver()) -> cerl:c_module(). @@ -237,6 +253,11 @@ lookup_mod_code(Mod, CS) when is_atom(Mod) -> lookup_mfa_code({_M, _F, _A} = MFA, CS) -> table__lookup(CS#codeserver.code, MFA). +-spec lookup_mfa_var_label(mfa(), codeserver()) -> {cerl:c_var(), label()}. + +lookup_mfa_var_label({_M, _F, _A} = MFA, CS) -> + ets:lookup_element(CS#codeserver.code, MFA, 3). + -spec get_next_core_label(codeserver()) -> label(). get_next_core_label(#codeserver{next_core_label = NCL}) -> @@ -251,8 +272,8 @@ set_next_core_label(NCL, CS) -> lookup_mod_records(Mod, #codeserver{records = RecDict}) when is_atom(Mod) -> case ets_dict_find(Mod, RecDict) of - error -> dict:new(); - {ok, Dict} -> Dict + error -> maps:new(); + {ok, Map} -> Map end. -spec get_records(codeserver()) -> mod_records(). @@ -262,11 +283,11 @@ get_records(#codeserver{records = RecDict}) -> -spec store_temp_records(module(), types(), codeserver()) -> codeserver(). -store_temp_records(Mod, Dict, #codeserver{temp_records = TempRecDict} = CS) +store_temp_records(Mod, Map, #codeserver{temp_records = TempRecDict} = CS) when is_atom(Mod) -> - case dict:size(Dict) =:= 0 of + case maps:size(Map) =:= 0 of true -> CS; - false -> CS#codeserver{temp_records = ets_dict_store(Mod, Dict, TempRecDict)} + false -> CS#codeserver{temp_records = ets_map_store(Mod, Map, TempRecDict)} end. -spec get_temp_records(codeserver()) -> mod_records(). @@ -284,20 +305,20 @@ set_temp_records(Dict, CS) -> -spec finalize_records(mod_records(), codeserver()) -> codeserver(). -finalize_records(Dict, CS) -> - true = ets:delete(CS#codeserver.temp_records), - Records = ets_read_concurrent_table(dialyzer_codeserver_records), +finalize_records(Dict, #codeserver{temp_records = TmpRecords, + records = Records} = CS) -> + true = ets:delete(TmpRecords), true = ets_dict_store_dict(Dict, Records), - CS#codeserver{records = Records, temp_records = clean}. + CS#codeserver{temp_records = clean}. -spec lookup_mod_contracts(atom(), codeserver()) -> contracts(). lookup_mod_contracts(Mod, #codeserver{contracts = ContDict}) when is_atom(Mod) -> case ets_dict_find(Mod, ContDict) of - error -> dict:new(); + error -> maps:new(); {ok, Keys} -> - dict:from_list([get_file_contract(Key, ContDict)|| Key <- Keys]) + maps:from_list([get_file_contract(Key, ContDict)|| Key <- Keys]) end. get_file_contract(Key, ContDict) -> @@ -330,48 +351,69 @@ get_callbacks(#codeserver{callbacks = CallbDict}) -> -spec store_temp_contracts(module(), contracts(), contracts(), codeserver()) -> codeserver(). -store_temp_contracts(Mod, SpecDict, CallbackDict, +store_temp_contracts(Mod, SpecMap, CallbackMap, #codeserver{temp_contracts = Cn, temp_callbacks = Cb} = CS) when is_atom(Mod) -> - CS1 = - case dict:size(SpecDict) =:= 0 of - true -> CS; - false -> - CS#codeserver{temp_contracts = ets_dict_store(Mod, SpecDict, Cn)} - end, - case dict:size(CallbackDict) =:= 0 of - true -> CS1; - false -> - CS1#codeserver{temp_callbacks = ets_dict_store(Mod, CallbackDict, Cb)} - end. - --spec get_temp_contracts(codeserver()) -> {mod_contracts(), mod_contracts()}. + CS1 = CS#codeserver{temp_contracts = ets_map_store(Mod, SpecMap, Cn)}, + CS1#codeserver{temp_callbacks = ets_map_store(Mod, CallbackMap, Cb)}. -get_temp_contracts(#codeserver{temp_contracts = TempContDict, - temp_callbacks = TempCallDict}) -> - {ets_dict_to_dict(TempContDict), ets_dict_to_dict(TempCallDict)}. +-spec contracts_modules(codeserver()) -> [module()]. --spec finalize_contracts(mod_contracts(), mod_contracts(), codeserver()) -> - codeserver(). +contracts_modules(#codeserver{temp_contracts = TempContTable}) -> + ets:select(TempContTable, [{{'$1', '$2'}, [], ['$1']}]). -finalize_contracts(SpecDict, CallbackDict, CS) -> - Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts), - Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks), - Contracts = dict:fold(fun decompose_spec_dict/3, Contracts, SpecDict), - Callbacks = dict:fold(fun decompose_cb_dict/3, Callbacks, CallbackDict), - CS#codeserver{contracts = Contracts, callbacks = Callbacks, - temp_contracts = clean, temp_callbacks = clean}. +-spec store_contracts(module(), contracts(), contracts(), codeserver()) -> + codeserver(). -decompose_spec_dict(Mod, Dict, Table) -> - Keys = dict:fetch_keys(Dict), - true = ets:insert(Table, dict:to_list(Dict)), - true = ets:insert(Table, {Mod, Keys}), - Table. +store_contracts(Mod, SpecMap, CallbackMap, CS) -> + #codeserver{contracts = SpecDict, callbacks = CallbackDict} = CS, + Keys = maps:keys(SpecMap), + true = ets:insert(SpecDict, maps:to_list(SpecMap)), + true = ets:insert(SpecDict, {Mod, Keys}), + true = ets:insert(CallbackDict, maps:to_list(CallbackMap)), + CS. -decompose_cb_dict(_Mod, Dict, Table) -> - true = ets:insert(Table, dict:to_list(Dict)), - Table. +-spec get_temp_contracts(module(), codeserver()) -> + {contracts(), contracts()}. + +get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict, + temp_callbacks = TempCallDict}) -> + [{Mod, Contracts}] = ets:lookup(TempContDict, Mod), + true = ets:delete(TempContDict, Mod), + [{Mod, Callbacks}] = ets:lookup(TempCallDict, Mod), + true = ets:delete(TempCallDict, Mod), + {Contracts, Callbacks}. + +-spec give_away(codeserver(), pid()) -> 'ok'. + +give_away(#codeserver{temp_records = TempRecords, + temp_contracts = TempContracts, + temp_callbacks = TempCallbacks, + records = Records, + contracts = Contracts, + callbacks = Callbacks}, Pid) -> + _ = [true = ets:give_away(Table, Pid, any) || + Table <- [TempRecords, TempContracts, TempCallbacks, + Records, Contracts, Callbacks], + Table =/= clean], + ok. + +-spec finalize_contracts(codeserver()) -> codeserver(). + +finalize_contracts(#codeserver{temp_contracts = TempContDict, + temp_callbacks = TempCallDict} = CS) -> + true = ets:delete(TempContDict), + true = ets:delete(TempCallDict), + CS#codeserver{temp_contracts = clean, temp_callbacks = clean}. + +-spec translate_fake_file(codeserver(), module(), file:filename()) -> + file:filename(). + +translate_fake_file(#codeserver{code = Code}, Module, FakeFile) -> + Files = ets:lookup_element(Code, {mod, Module}, 2), + {FakeFile, File} = lists:keyfind(FakeFile, 1, Files), + File. table__lookup(TablePid, M) when is_atom(M) -> {Name, Exports, Attrs, Keys, As} = ets:lookup_element(TablePid, M, 2), @@ -379,3 +421,25 @@ table__lookup(TablePid, M) when is_atom(M) -> cerl:ann_c_module(As, Name, Exports, Attrs, Defs); table__lookup(TablePid, MFA) -> ets:lookup_element(TablePid, MFA, 2). + +compress_file_anno(Term) -> + {Files, SmallTerm} = compress_file_anno(Term, []), + {[{FakeFile, File} || {File, {file, FakeFile}} <- Files], SmallTerm}. + +compress_file_anno({file, F}, Fs) when is_list(F) -> + case lists:keyfind(F, 1, Fs) of + false -> + I = integer_to_list(length(Fs)), + FileI = {file, I}, + NFs = [{F, FileI}|Fs], + {NFs, FileI}; + {F, FileI} -> {Fs, FileI} + end; +compress_file_anno(T, Fs) when is_tuple(T) -> + {NFs, NL} = compress_file_anno(tuple_to_list(T), Fs), + {NFs, list_to_tuple(NL)}; +compress_file_anno([E|L], Fs) -> + {Fs1, NE} = compress_file_anno(E, Fs), + {NFs, NL} = compress_file_anno(L, Fs1), + {NFs, [NE|NL]}; +compress_file_anno(T, Fs) -> {Fs, T}. diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 73b04b305b..f3fba68e84 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -31,7 +31,7 @@ get_contract_return/2, %% get_contract_signature/1, is_overloaded/1, - process_contract_remote_types/1, + process_contract_remote_types/2, store_tmp_contract/5]). -export_type([file_contract/0, plt_contracts/0]). @@ -146,14 +146,13 @@ sequence([], _Delimiter) -> ""; sequence([H], _Delimiter) -> H; sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter). --spec process_contract_remote_types(dialyzer_codeserver:codeserver()) -> - dialyzer_codeserver:codeserver(). +-spec process_contract_remote_types(dialyzer_codeserver:codeserver(), + erl_types:mod_records()) -> + dialyzer_codeserver:codeserver(). -process_contract_remote_types(CodeServer) -> - {TmpContractDict, TmpCallbackDict} = - dialyzer_codeserver:get_temp_contracts(CodeServer), +process_contract_remote_types(CodeServer, RecordDict) -> + Mods = dialyzer_codeserver:contracts_modules(CodeServer), ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer), - RecordDict = dialyzer_codeserver:get_records(CodeServer), ContractFun = fun({{_M, _F, _A}=MFA, {File, TmpContract, Xtra}}, C0) -> #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract, @@ -165,20 +164,21 @@ process_contract_remote_types(CodeServer) -> {{MFA, {File, Contract, Xtra}}, C2} end, ModuleFun = - fun({ModuleName, ContractDict}, C3) -> - {NewContractList, C4} = - lists:mapfoldl(ContractFun, C3, dict:to_list(ContractDict)), - {{ModuleName, dict:from_list(NewContractList)}, C4} + fun(ModuleName) -> + Cache = erl_types:cache__new(), + {ContractMap, CallbackMap} = + dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer), + {NewContractList, Cache1} = + lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)), + {NewCallbackList, _NewCache} = + lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)), + dialyzer_codeserver:store_contracts(ModuleName, + maps:from_list(NewContractList), + maps:from_list(NewCallbackList), + CodeServer) end, - Cache = erl_types:cache__new(), - {NewContractList, C5} = - lists:mapfoldl(ModuleFun, Cache, dict:to_list(TmpContractDict)), - {NewCallbackList, _C6} = - lists:mapfoldl(ModuleFun, C5, dict:to_list(TmpCallbackDict)), - NewContractDict = dict:from_list(NewContractList), - NewCallbackDict = dict:from_list(NewCallbackList), - dialyzer_codeserver:finalize_contracts(NewContractDict, NewCallbackDict, - CodeServer). + lists:foreach(ModuleFun, Mods), + dialyzer_codeserver:finalize_contracts(CodeServer). -type opaques_fun() :: fun((module()) -> [erl_types:erl_type()]). @@ -397,7 +397,7 @@ solve_constraints(Contract, Call, Constraints) -> %% ?debug("Inf: ~s\n", [erl_types:t_to_string(Inf)]), %% erl_types:t_assign_variables_to_subtype(Contract, Inf). --type contracts() :: dict:dict(mfa(),dialyzer_contracts:file_contract()). +-type contracts() :: dialyzer_codeserver:contracts(). %% Checks the contracts for functions that are not implemented -spec contracts_without_fun(contracts(), [_], dialyzer_callgraph:callgraph()) -> @@ -407,12 +407,12 @@ contracts_without_fun(Contracts, AllFuns0, Callgraph) -> AllFuns1 = [{dialyzer_callgraph:lookup_name(Label, Callgraph), Arity} || {Label, Arity} <- AllFuns0], AllFuns2 = [{M, F, A} || {{ok, {M, F, _}}, A} <- AllFuns1], - AllContractMFAs = dict:fetch_keys(Contracts), + AllContractMFAs = maps:keys(Contracts), ErrorContractMFAs = AllContractMFAs -- AllFuns2, [warn_spec_missing_fun(MFA, Contracts) || MFA <- ErrorContractMFAs]. warn_spec_missing_fun({M, F, A} = MFA, Contracts) -> - {{File, Line}, _Contract, _Xtra} = dict:fetch(MFA, Contracts), + {{File, Line}, _Contract, _Xtra} = maps:get(MFA, Contracts), WarningInfo = {File, Line, MFA}, {?WARN_CONTRACT_SYNTAX, WarningInfo, {spec_missing_fun, [M, F, A]}}. @@ -445,11 +445,11 @@ insert_constraints([], Map) -> Map. -spec store_tmp_contract(mfa(), file_line(), spec_data(), contracts(), types()) -> contracts(). -store_tmp_contract(MFA, FileLine, {TypeSpec, Xtra}, SpecDict, RecordsDict) -> +store_tmp_contract(MFA, FileLine, {TypeSpec, Xtra}, SpecMap, RecordsDict) -> %% io:format("contract from form: ~p\n", [TypeSpec]), TmpContract = contract_from_form(TypeSpec, MFA, RecordsDict, FileLine), %% io:format("contract: ~p\n", [TmpContract]), - dict:store(MFA, {FileLine, TmpContract, Xtra}, SpecDict). + maps:put(MFA, {FileLine, TmpContract, Xtra}, SpecMap). contract_from_form(Forms, MFA, RecDict, FileLine) -> {CFuns, Forms1} = contract_from_form(Forms, MFA, RecDict, FileLine, [], []), @@ -677,7 +677,7 @@ get_invalid_contract_warnings(Modules, CodeServer, Plt, FindOpaques) -> get_invalid_contract_warnings_modules([Mod|Mods], CodeServer, Plt, FindOpaques, Acc) -> Contracts1 = dialyzer_codeserver:lookup_mod_contracts(Mod, CodeServer), - Contracts2 = dict:to_list(Contracts1), + Contracts2 = maps:to_list(Contracts1), Records = dialyzer_codeserver:lookup_mod_records(Mod, CodeServer), NewAcc = get_invalid_contract_warnings_funs(Contracts2, Plt, Records, FindOpaques, Acc), get_invalid_contract_warnings_modules(Mods, CodeServer, Plt, FindOpaques, NewAcc); diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl index 639ed426df..ce292e1140 100644 --- a/lib/dialyzer/src/dialyzer_dataflow.erl +++ b/lib/dialyzer/src/dialyzer_dataflow.erl @@ -529,7 +529,7 @@ handle_apply_or_call([{TypeOfApply, {Fun, Sig, Contr, LocalRet}}|Left], case is_race_analysis_enabled(State) of true -> Ann = cerl:get_ann(Tree), - File = get_file(Ann), + File = get_file(Ann, State), Line = abs(get_line(Ann)), dialyzer_races:store_race_call(Fun, ArgTypes, Args, {File, Line}, State); @@ -3090,7 +3090,7 @@ state__add_warning(#state{warnings = Warnings, warning_mode = true} = State, Ann = cerl:get_ann(Tree), case Force of true -> - WarningInfo = {get_file(Ann), + WarningInfo = {get_file(Ann, State), abs(get_line(Ann)), State#state.curr_fun}, Warn = {Tag, WarningInfo, Msg}, @@ -3100,7 +3100,9 @@ state__add_warning(#state{warnings = Warnings, warning_mode = true} = State, case is_compiler_generated(Ann) of true -> State; false -> - WarningInfo = {get_file(Ann), get_line(Ann), State#state.curr_fun}, + WarningInfo = {get_file(Ann, State), + get_line(Ann), + State#state.curr_fun}, Warn = {Tag, WarningInfo, Msg}, case Tag of ?WARN_CONTRACT_RANGE -> ok; @@ -3499,6 +3501,12 @@ state__put_races(Races, State) -> state__records_only(#state{records = Records}) -> #state{records = Records}. +-spec state__translate_file(file:filename(), state()) -> file:filename(). + +state__translate_file(FakeFile, State) -> + #state{codeserver = CodeServer, module = Module} = State, + dialyzer_codeserver:translate_fake_file(CodeServer, Module, FakeFile). + %%% =========================================================================== %%% %%% Races @@ -3570,9 +3578,11 @@ get_line([Line|_]) when is_integer(Line) -> Line; get_line([_|Tail]) -> get_line(Tail); get_line([]) -> -1. -get_file([]) -> []; -get_file([{file, File}|_]) -> File; -get_file([_|Tail]) -> get_file(Tail). +get_file([], _State) -> []; +get_file([{file, FakeFile}|_], State) -> + state__translate_file(FakeFile, State); +get_file([_|Tail], State) -> + get_file(Tail, State). is_compiler_generated(Ann) -> lists:member(compiler_generated, Ann) orelse (get_line(Ann) < 1). diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl index 4caf64d007..1701aff2f2 100644 --- a/lib/dialyzer/src/dialyzer_gui_wx.erl +++ b/lib/dialyzer/src/dialyzer_gui_wx.erl @@ -505,8 +505,9 @@ gui_loop(#gui_state{backend_pid = BackendPid, doc_plt = DocPlt, end, ExplanationPid = spawn_link(Fun), gui_loop(State#gui_state{expl_pid = ExplanationPid}); - {BackendPid, done, _NewPlt, NewDocPlt} -> + {BackendPid, done, NewMiniPlt, NewDocPlt} -> message(State, "Analysis done"), + dialyzer_plt:delete(NewMiniPlt), config_gui_stop(State), gui_loop(State#gui_state{doc_plt = NewDocPlt}); {'EXIT', BackendPid, {error, Reason}} -> diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index cf2f0e919e..0eda73a208 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -58,7 +58,9 @@ get_specs/4, to_file/4, get_mini_plt/1, - restore_full_plt/2 + restore_full_plt/1, + delete/1, + give_away/2 ]). %% Debug utilities @@ -82,14 +84,16 @@ %%---------------------------------------------------------------------- -record(plt, {info = table_new() :: dict:dict(), - types = table_new() :: dict:dict(), + types = table_new() :: erl_types:mod_records(), contracts = table_new() :: dict:dict(), callbacks = table_new() :: dict:dict(), exported_types = sets:new() :: sets:set()}). -record(mini_plt, {info :: ets:tid(), + types :: ets:tid(), contracts :: ets:tid(), - callbacks :: ets:tid() + callbacks :: ets:tid(), + exported_types :: ets:tid() }). -opaque plt() :: #plt{} | #mini_plt{}. @@ -130,6 +134,10 @@ delete_module(#plt{info = Info, types = Types, -spec delete_list(plt(), [mfa() | integer()]) -> plt(). +delete_list(#mini_plt{info = Info, + contracts = Contracts}=Plt, List) -> + Plt#mini_plt{info = ets_table_delete_list(Info, List), + contracts = ets_table_delete_list(Contracts, List)}; delete_list(#plt{info = Info, types = Types, contracts = Contracts, callbacks = Callbacks, @@ -183,7 +191,7 @@ lookup(Plt, Label) when is_integer(Label) -> lookup_1(#mini_plt{info = Info}, MFAorLabel) -> ets_table_lookup(Info, MFAorLabel). --spec insert_types(plt(), dict:dict()) -> plt(). +-spec insert_types(plt(), erl_types:mod_records()) -> plt(). insert_types(PLT, Rec) -> PLT#plt{types = Rec}. @@ -193,7 +201,7 @@ insert_types(PLT, Rec) -> insert_exported_types(PLT, Set) -> PLT#plt{exported_types = Set}. --spec get_types(plt()) -> dict:dict(). +-spec get_types(plt()) -> erl_types:mod_records(). get_types(#plt{types = Types}) -> Types. @@ -253,8 +261,10 @@ from_file(FileName, ReturnInfo) -> Msg = io_lib:format("Old PLT file ~s\n", [FileName]), plt_error(Msg); ok -> + Types = [{Mod, maps:from_list(dict:to_list(Types))} || + {Mod, Types} <- dict:to_list(Rec#file_plt.types)], Plt = #plt{info = Rec#file_plt.info, - types = Rec#file_plt.types, + types = dict:from_list(Types), contracts = Rec#file_plt.contracts, callbacks = Rec#file_plt.callbacks, exported_types = Rec#file_plt.exported_types}, @@ -371,12 +381,14 @@ to_file(FileName, end, OldModDeps, ModDeps), ImplMd5 = compute_implementation_md5(), + FileTypes = dict:from_list([{Mod, dict:from_list(maps:to_list(MTypes))} || + {Mod, MTypes} <- dict:to_list(Types)]), Record = #file_plt{version = ?VSN, file_md5_list = MD5, info = Info, contracts = Contracts, callbacks = Callbacks, - types = Types, + types = FileTypes, exported_types = ExpTypes, mod_deps = NewModDeps, implementation_md5 = ImplMd5}, @@ -510,32 +522,100 @@ init_md5_list_1(Md5List, [], Acc) -> -spec get_mini_plt(plt()) -> plt(). -get_mini_plt(#plt{info = Info, contracts = Contracts, callbacks = Callbacks}) -> - [ETSInfo, ETSContracts, ETSCallbacks] = - [ets:new(Name, [public]) || Name <- [plt_info, plt_contracts, plt_callbacks]], +get_mini_plt(#plt{info = Info, + types = Types, + contracts = Contracts, + callbacks = Callbacks, + exported_types = ExpTypes}) -> + [ETSInfo, ETSTypes, ETSContracts, ETSCallbacks, ETSExpTypes] = + [ets:new(Name, [public]) || + Name <- [plt_info, plt_types, plt_contracts, plt_callbacks, + plt_exported_types]], CallbackList = dict:to_list(Callbacks), CallbacksByModule = [{M, [Cb || {{M1,_,_},_} = Cb <- CallbackList, M1 =:= M]} || M <- lists:usort([M || {{M,_,_},_} <- CallbackList])], - [true, true] = + [true, true, true] = [ets:insert(ETS, dict:to_list(Data)) || - {ETS, Data} <- [{ETSInfo, Info}, {ETSContracts, Contracts}]], + {ETS, Data} <- [{ETSInfo, Info}, + {ETSTypes, Types}, + {ETSContracts, Contracts}]], true = ets:insert(ETSCallbacks, CallbacksByModule), - #mini_plt{info = ETSInfo, contracts = ETSContracts, callbacks = ETSCallbacks}; + true = ets:insert(ETSExpTypes, [{ET} || ET <- sets:to_list(ExpTypes)]), + #mini_plt{info = ETSInfo, + types = ETSTypes, + contracts = ETSContracts, + callbacks = ETSCallbacks, + exported_types = ETSExpTypes}; get_mini_plt(undefined) -> undefined. --spec restore_full_plt(plt(), plt()) -> plt(). - -restore_full_plt(#mini_plt{info = ETSInfo, contracts = ETSContracts}, Plt) -> - Info = dict:from_list(ets:tab2list(ETSInfo)), - Contracts = dict:from_list(ets:tab2list(ETSContracts)), - ets:delete(ETSContracts), - ets:delete(ETSInfo), - Plt#plt{info = Info, contracts = Contracts}; -restore_full_plt(undefined, undefined) -> +-spec restore_full_plt(plt()) -> plt(). + +restore_full_plt(#mini_plt{info = ETSInfo, + types = ETSTypes, + contracts = ETSContracts, + callbacks = ETSCallbacks, + exported_types = ETSExpTypes} = MiniPlt) -> + Info = dict:from_list(tab2list(ETSInfo)), + Contracts = dict:from_list(tab2list(ETSContracts)), + Types = dict:from_list(tab2list(ETSTypes)), + Callbacks = + dict:from_list([Cb || {_M, Cbs} <- tab2list(ETSCallbacks), Cb <- Cbs]), + ExpTypes = sets:from_list([E || {E} <- tab2list(ETSExpTypes)]), + ok = delete(MiniPlt), + #plt{info = Info, + types = Types, + contracts = Contracts, + callbacks = Callbacks, + exported_types = ExpTypes}; +restore_full_plt(undefined) -> undefined. +-spec delete(plt()) -> 'ok'. + +delete(#mini_plt{info = ETSInfo, + types = ETSTypes, + contracts = ETSContracts, + callbacks = ETSCallbacks, + exported_types = ETSExpTypes}) -> + true = ets:delete(ETSContracts), + true = ets:delete(ETSTypes), + true = ets:delete(ETSInfo), + true = ets:delete(ETSCallbacks), + true = ets:delete(ETSExpTypes), + ok. + +-spec give_away(plt(), pid()) -> 'ok'. + +give_away(#mini_plt{info = ETSInfo, + types = ETSTypes, + contracts = ETSContracts, + callbacks = ETSCallbacks, + exported_types = ETSExpTypes}, + Pid) -> + true = ets:give_away(ETSContracts, Pid, any), + true = ets:give_away(ETSTypes, Pid, any), + true = ets:give_away(ETSInfo, Pid, any), + true = ets:give_away(ETSCallbacks, Pid, any), + true = ets:give_away(ETSExpTypes, Pid, any), + ok. + +%% Somewhat slower than ets:tab2list(), but uses less memory. +tab2list(T) -> + tab2list(ets:first(T), T, []). + +tab2list('$end_of_table', T, A) -> + case ets:first(T) of % no safe_fixtable()... + '$end_of_table' -> A; + Key -> tab2list(Key, T, A) + end; +tab2list(Key, T, A) -> + Vs = ets:lookup(T, Key), + Key1 = ets:next(T, Key), + ets:delete(T, Key), + tab2list(Key1, T, Vs ++ A). + %%--------------------------------------------------------------------------- %% Edoc @@ -607,6 +687,12 @@ table_delete_module1(Plt, Mod) -> table_delete_module2(Plt, Mod) -> dict:filter(fun(M, _Val) -> M =/= Mod end, Plt). +ets_table_delete_list(Tab, [H|T]) -> + ets:delete(Tab, H), + ets_table_delete_list(Tab, T); +ets_table_delete_list(Tab, []) -> + Tab. + table_delete_list(Plt, [H|T]) -> table_delete_list(dict:erase(H, Plt), T); table_delete_list(Plt, []) -> diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl index 987da3aecf..0e44a5223f 100644 --- a/lib/dialyzer/src/dialyzer_succ_typings.erl +++ b/lib/dialyzer/src/dialyzer_succ_typings.erl @@ -96,7 +96,7 @@ analyze_callgraph(Callgraph, Plt, Codeserver, TimingServer, Solvers, Parent) -> NewState = init_state_and_get_success_typings(Callgraph, Plt, Codeserver, TimingServer, Solvers, Parent), - dialyzer_plt:restore_full_plt(NewState#st.plt, Plt). + NewState#st.plt. %%-------------------------------------------------------------------- @@ -111,6 +111,7 @@ init_state_and_get_success_typings(Callgraph, Plt, Codeserver, get_refined_success_typings(SCCs, #st{callgraph = Callgraph, timing_server = TimingServer} = State) -> + erlang:garbage_collect(), case find_succ_typings(SCCs, State) of {fixpoint, State1} -> State1; {not_fixpoint, NotFixpoint1, State1} -> @@ -155,8 +156,8 @@ get_warnings(Callgraph, Plt, DocPlt, Codeserver, ?timing(TimingServer, "warning", get_warnings_from_modules(Mods, InitState, MiniDocPlt)), {postprocess_warnings(CWarns ++ ModWarns, Codeserver), - dialyzer_plt:restore_full_plt(MiniPlt, Plt), - dialyzer_plt:restore_full_plt(MiniDocPlt, DocPlt)}. + MiniPlt, + dialyzer_plt:restore_full_plt(MiniDocPlt)}. get_warnings_from_modules(Mods, State, DocPlt) -> #st{callgraph = Callgraph, codeserver = Codeserver, @@ -174,10 +175,10 @@ collect_warnings(M, {Codeserver, Callgraph, Plt, DocPlt}) -> %% Check if there are contracts for functions that do not exist Warnings1 = dialyzer_contracts:contracts_without_fun(Contracts, AllFuns, Callgraph), + Attrs = cerl:module_attrs(ModCode), {Warnings2, FunTypes} = dialyzer_dataflow:get_warnings(ModCode, Plt, Callgraph, Codeserver, Records), - Attrs = cerl:module_attrs(ModCode), Warnings3 = dialyzer_behaviours:check_callbacks(M, Attrs, Records, Plt, Codeserver), DocPlt = insert_into_doc_plt(FunTypes, Callgraph, DocPlt), @@ -262,7 +263,7 @@ refine_one_module(M, {CodeServer, Callgraph, Plt, _Solvers}) -> NewFunTypes = dialyzer_dataflow:get_fun_types(ModCode, Plt, Callgraph, CodeServer, Records), Contracts1 = dialyzer_codeserver:lookup_mod_contracts(M, CodeServer), - Contracts = orddict:from_list(dict:to_list(Contracts1)), + Contracts = orddict:from_list(maps:to_list(Contracts1)), FindOpaques = find_opaques_fun(Records), DecoratedFunTypes = decorate_succ_typings(Contracts, Callgraph, NewFunTypes, FindOpaques), @@ -348,21 +349,25 @@ find_succ_typings(SCCs, #st{codeserver = Codeserver, callgraph = Callgraph, -spec find_succ_types_for_scc(scc(), typesig_init_data()) -> [mfa_or_funlbl()]. -find_succ_types_for_scc(SCC, {Codeserver, Callgraph, Plt, Solvers}) -> - SCC_Info = [{MFA, - dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver), - dialyzer_codeserver:lookup_mod_records(M, Codeserver)} - || {M, _, _} = MFA <- SCC], +find_succ_types_for_scc(SCC0, {Codeserver, Callgraph, Plt, Solvers}) -> + SCC = [MFA || {_, _, _} = MFA <- SCC0], Contracts1 = [{MFA, dialyzer_codeserver:lookup_mfa_contract(MFA, Codeserver)} - || {_, _, _} = MFA <- SCC], + || MFA <- SCC], Contracts2 = [{MFA, Contract} || {MFA, {ok, Contract}} <- Contracts1], Contracts3 = orddict:from_list(Contracts2), Label = dialyzer_codeserver:get_next_core_label(Codeserver), - AllFuns = collect_fun_info([Fun || {_MFA, {_Var, Fun}, _Rec} <- SCC_Info]), + AllFuns = lists:append( + [begin + {_Var, Fun} = + dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver), + collect_fun_info([Fun]) + end || MFA <- SCC]), + erlang:garbage_collect(), PropTypes = get_fun_types_from_plt(AllFuns, Callgraph, Plt), %% Assume that the PLT contains the current propagated types - FunTypes = dialyzer_typesig:analyze_scc(SCC_Info, Label, Callgraph, - Plt, PropTypes, Solvers), + FunTypes = dialyzer_typesig:analyze_scc(SCC, Label, Callgraph, + Codeserver, Plt, PropTypes, + Solvers), AllFunSet = sets:from_list([X || {X, _} <- AllFuns]), FilteredFunTypes = dict:filter(fun(X, _) -> sets:is_element(X, AllFunSet) end, FunTypes), diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index 1787b66192..a9ebac6c8b 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -29,7 +29,7 @@ -module(dialyzer_typesig). --export([analyze_scc/6]). +-export([analyze_scc/7]). -export([get_safe_underapprox/2]). %%-import(helper, %% 'helper' could be any module doing sanity checks... @@ -101,10 +101,9 @@ -type types() :: erl_types:type_table(). --type typesig_scc() :: [{mfa(), {cerl:c_var(), cerl:c_fun()}, types()}]. -type typesig_funmap() :: #{type_var() => type_var()}. --type prop_types() :: dict:dict(label(), types()). +-type prop_types() :: dict:dict(label(), erl_types:erl_type()). -record(state, {callgraph :: dialyzer_callgraph:callgraph() | 'undefined', @@ -121,7 +120,7 @@ plt :: dialyzer_plt:plt() | 'undefined', prop_types = dict:new() :: prop_types(), - records = dict:new() :: types(), + records = maps:new() :: types(), scc = [] :: ordsets:ordset(type_var()), mfas :: [mfa()], solvers = [] :: [solver()] @@ -160,11 +159,10 @@ %%----------------------------------------------------------------------------- %% Analysis of strongly connected components. %% -%% analyze_scc(SCC, NextLabel, CallGraph, PLT, PropTypes, Solvers) -> FunTypes +%% analyze_scc(SCC, NextLabel, CallGraph, CodeServer, +%% PLT, PropTypes, Solvers) -> FunTypes %% -%% SCC - [{MFA, Def, Records}] -%% where Def = {Var, Fun} as in the Core Erlang module definitions. -%% Records = dict(RecName, {Arity, [{FieldName, FieldType}]}) +%% SCC - [{MFA}] %% NextLabel - An integer that is higher than any label in the code. %% CallGraph - A callgraph as produced by dialyzer_callgraph.erl %% Note: The callgraph must have been built with all the @@ -176,28 +174,27 @@ %% Solvers - User specified solvers. %%----------------------------------------------------------------------------- --spec analyze_scc(typesig_scc(), label(), +-spec analyze_scc([mfa()], label(), dialyzer_callgraph:callgraph(), + dialyzer_codeserver:codeserver(), dialyzer_plt:plt(), prop_types(), [solver()]) -> prop_types(). -analyze_scc(SCC, NextLabel, CallGraph, Plt, PropTypes, Solvers0) -> +analyze_scc(SCC, NextLabel, CallGraph, CServer, Plt, PropTypes, Solvers0) -> Solvers = solvers(Solvers0), - assert_format_of_scc(SCC), - State1 = new_state(SCC, NextLabel, CallGraph, Plt, PropTypes, Solvers), - DefSet = add_def_list([Var || {_MFA, {Var, _Fun}, _Rec} <- SCC], sets:new()), - State2 = traverse_scc(SCC, DefSet, State1), + State1 = new_state(SCC, NextLabel, CallGraph, CServer, Plt, PropTypes, + Solvers), + DefSet = add_def_list(maps:values(State1#state.name_map), sets:new()), + ModRecs = [{M, dialyzer_codeserver:lookup_mod_records(M, CServer)} || + M <- lists:usort([M || {M, _, _} <- SCC])], + State2 = traverse_scc(SCC, CServer, DefSet, ModRecs, State1), State3 = state__finalize(State2), + erlang:garbage_collect(), Funs = state__scc(State3), pp_constrs_scc(Funs, State3), constraints_to_dot_scc(Funs, State3), T = solve(Funs, State3), dict:from_list(maps:to_list(T)). -assert_format_of_scc([{_MFA, {_Var, _Fun}, _Records}|Left]) -> - assert_format_of_scc(Left); -assert_format_of_scc([]) -> - ok. - solvers([]) -> [v2]; solvers(Solvers) -> Solvers. @@ -207,12 +204,14 @@ solvers(Solvers) -> Solvers. %% %% ============================================================================ -traverse_scc([{_MFA, Def, Rec}|Left], DefSet, AccState) -> +traverse_scc([{M,_,_}=MFA|Left], Codeserver, DefSet, ModRecs, AccState) -> + Def = dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver), + {M, Rec} = lists:keyfind(M, 1, ModRecs), TmpState1 = state__set_rec_dict(AccState, Rec), DummyLetrec = cerl:c_letrec([Def], cerl:c_atom(foo)), {NewAccState, _} = traverse(DummyLetrec, DefSet, TmpState1), - traverse_scc(Left, DefSet, NewAccState); -traverse_scc([], _DefSet, AccState) -> + traverse_scc(Left, Codeserver, DefSet, ModRecs, NewAccState); +traverse_scc([], _Codeserver, _DefSet, _ModRecs, AccState) -> AccState. traverse(Tree, DefinedVars, State) -> @@ -2702,11 +2701,14 @@ pp_map(_S, _Map) -> %% %% ============================================================================ -new_state(SCC0, NextLabel, CallGraph, Plt, PropTypes, Solvers) -> - List = [{MFA, Var} || {MFA, {Var, _Fun}, _Rec} <- SCC0], +new_state(MFAs, NextLabel, CallGraph, CServer, Plt, PropTypes, Solvers) -> + List_SCC = + [begin + {Var, Label} = dialyzer_codeserver:lookup_mfa_var_label(MFA, CServer), + {{MFA, Var}, t_var(Label)} + end || MFA <- MFAs], + {List, SCC} = lists:unzip(List_SCC), NameMap = maps:from_list(List), - MFAs = [MFA || {MFA, _Var} <- List], - SCC = [mk_var(Fun) || {_MFA, {_Var, Fun}, _Rec} <- SCC0], SelfRec = case SCC of [OneF] -> diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 1f2d3e3aaa..e71a953279 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -202,7 +202,7 @@ get_core_from_abstract_code(AbstrCode, Opts) -> get_record_and_type_info(AbstractCode) -> Module = get_module(AbstractCode), - get_record_and_type_info(AbstractCode, Module, dict:new()). + get_record_and_type_info(AbstractCode, Module, maps:new()). -spec get_record_and_type_info(abstract_code(), module(), type_table()) -> {'ok', type_table()} | {'error', string()}. @@ -215,7 +215,7 @@ get_record_and_type_info([{attribute, A, record, {Name, Fields0}}|Left], {ok, Fields} = get_record_fields(Fields0, RecDict), Arity = length(Fields), FN = {File, erl_anno:line(A)}, - NewRecDict = dict:store({record, Name}, {FN, [{Arity,Fields}]}, RecDict), + NewRecDict = maps:put({record, Name}, {FN, [{Arity,Fields}]}, RecDict), get_record_and_type_info(Left, Module, NewRecDict, File); get_record_and_type_info([{attribute, A, type, {{record, Name}, Fields0, []}} |Left], Module, RecDict, File) -> @@ -223,7 +223,7 @@ get_record_and_type_info([{attribute, A, type, {{record, Name}, Fields0, []}} {ok, Fields} = get_record_fields(Fields0, RecDict), Arity = length(Fields), FN = {File, erl_anno:line(A)}, - NewRecDict = dict:store({record, Name}, {FN, [{Arity, Fields}]}, RecDict), + NewRecDict = maps:put({record, Name}, {FN, [{Arity, Fields}]}, RecDict), get_record_and_type_info(Left, Module, NewRecDict, File); get_record_and_type_info([{attribute, A, Attr, {Name, TypeForm}}|Left], Module, RecDict, File) @@ -263,9 +263,9 @@ add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, FN, false -> try erl_types:t_var_names(ArgForms) of ArgNames -> - dict:store({TypeOrOpaque, Name, Arity}, - {{Module, FN, TypeForm, ArgNames}, - erl_types:t_any()}, RecDict) + maps:put({TypeOrOpaque, Name, Arity}, + {{Module, FN, TypeForm, ArgNames}, + erl_types:t_any()}, RecDict) catch _:_ -> throw({error, flat_format("Type declaration for ~w does not " @@ -296,19 +296,18 @@ get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) -> get_record_fields([], _RecDict, Acc) -> lists:reverse(Acc). --spec process_record_remote_types(codeserver()) -> codeserver(). +-spec process_record_remote_types(codeserver()) -> + {codeserver(), mod_records()}. %% The field types are cached. Used during analysis when handling records. process_record_remote_types(CServer) -> TempRecords = dialyzer_codeserver:get_temp_records(CServer), ExpTypes = dialyzer_codeserver:get_exported_types(CServer), - Cache = erl_types:cache__new(), - {TempRecords1, Cache1} = - process_opaque_types0(TempRecords, ExpTypes, Cache), + TempRecords1 = process_opaque_types0(TempRecords, ExpTypes), %% A cache (not the field type cache) is used for speeding things up a bit. VarTable = erl_types:var_table__new(), ModuleFun = - fun({Module, Record}, C0) -> + fun({Module, Record}) -> RecordFun = fun({Key, Value}, C2) -> case Key of @@ -334,24 +333,27 @@ process_record_remote_types(CServer) -> _Other -> {{Key, Value}, C2} end end, - {RecordList, C1} = - lists:mapfoldl(RecordFun, C0, dict:to_list(Record)), - {{Module, dict:from_list(RecordList)}, C1} + Cache = erl_types:cache__new(), + {RecordList, _NewCache} = + lists:mapfoldl(RecordFun, Cache, maps:to_list(Record)), + {Module, maps:from_list(RecordList)} end, - {NewRecordsList, C1} = - lists:mapfoldl(ModuleFun, Cache1, dict:to_list(TempRecords1)), + NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)), NewRecords = dict:from_list(NewRecordsList), - _C8 = check_record_fields(NewRecords, ExpTypes, C1), - dialyzer_codeserver:finalize_records(NewRecords, CServer). + check_record_fields(NewRecords, ExpTypes), + {dialyzer_codeserver:finalize_records(NewRecords, CServer), NewRecords}. %% erl_types:t_from_form() substitutes the declaration of opaque types %% for the expanded type in some cases. To make sure the initial type, %% any(), is not used, the expansion is done twice. %% XXX: Recursive opaque types are not handled well. -process_opaque_types0(TempRecords0, TempExpTypes, Cache) -> - {TempRecords1, NewCache} = +process_opaque_types0(TempRecords0, TempExpTypes) -> + Cache = erl_types:cache__new(), + {TempRecords1, Cache1} = process_opaque_types(TempRecords0, TempExpTypes, Cache), - process_opaque_types(TempRecords1, TempExpTypes, NewCache). + {TempRecords, _NewCache} = + process_opaque_types(TempRecords1, TempExpTypes, Cache1), + TempRecords. process_opaque_types(TempRecords, TempExpTypes, Cache) -> VarTable = erl_types:var_table__new(), @@ -371,8 +373,8 @@ process_opaque_types(TempRecords, TempExpTypes, Cache) -> end end, {RecordList, C1} = - lists:mapfoldl(RecordFun, C0, dict:to_list(Record)), - {{Module, dict:from_list(RecordList)}, C1} + lists:mapfoldl(RecordFun, C0, maps:to_list(Record)), + {{Module, maps:from_list(RecordList)}, C1} %% dict:map(RecordFun, Record) end, {TempRecordList, NewCache} = @@ -380,7 +382,8 @@ process_opaque_types(TempRecords, TempExpTypes, Cache) -> {dict:from_list(TempRecordList), NewCache}. %% dict:map(ModuleFun, TempRecords). -check_record_fields(Records, TempExpTypes, Cache) -> +check_record_fields(Records, TempExpTypes) -> + Cache = erl_types:cache__new(), VarTable = erl_types:var_table__new(), CheckFun = fun({Module, Element}, C0) -> @@ -410,9 +413,10 @@ check_record_fields(Records, TempExpTypes, Cache) -> msg_with_position(Fun, FileLine) end end, - lists:foldl(ElemFun, C0, dict:to_list(Element)) + lists:foldl(ElemFun, C0, maps:to_list(Element)) end, - lists:foldl(CheckFun, Cache, dict:to_list(Records)). + _NewCache = lists:foldl(CheckFun, Cache, dict:to_list(Records)), + ok. msg_with_position(Fun, FileLine) -> try Fun() @@ -435,17 +439,17 @@ merge_records(NewRecords, OldRecords) -> %% %% ============================================================================ --type spec_dict() :: dict:dict(). --type callback_dict() :: dict:dict(). +-type spec_map() :: dialyzer_codeserver:contracts(). +-type callback_map() :: dialyzer_codeserver:contracts(). -spec get_spec_info(module(), abstract_code(), type_table()) -> - {'ok', spec_dict(), callback_dict()} | {'error', string()}. + {'ok', spec_map(), callback_map()} | {'error', string()}. -get_spec_info(ModName, AbstractCode, RecordsDict) -> +get_spec_info(ModName, AbstractCode, RecordsMap) -> OptionalCallbacks0 = get_optional_callbacks(AbstractCode, ModName), OptionalCallbacks = gb_sets:from_list(OptionalCallbacks0), - get_spec_info(AbstractCode, dict:new(), dict:new(), - RecordsDict, ModName, OptionalCallbacks, "nofile"). + get_spec_info(AbstractCode, maps:new(), maps:new(), + RecordsMap, ModName, OptionalCallbacks, "nofile"). get_optional_callbacks(Abs, ModName) -> [{ModName, F, A} || {F, A} <- get_optional_callbacks(Abs)]. @@ -463,7 +467,7 @@ get_optional_callbacks(Abs) -> %% are erl_types:erl_type() get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left], - SpecDict, CallbackDict, RecordsDict, ModName, OptCb, File) + SpecMap, CallbackMap, RecordsMap, ModName, OptCb, File) when ((Contract =:= 'spec') or (Contract =:= 'callback')), is_list(TypeSpec) -> Ln = erl_anno:line(Anno), @@ -472,24 +476,24 @@ get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left], {F, A} -> {ModName, F, A} end, Xtra = [optional_callback || gb_sets:is_member(MFA, OptCb)], - ActiveDict = + ActiveMap = case Contract of - spec -> SpecDict; - callback -> CallbackDict + spec -> SpecMap; + callback -> CallbackMap end, - try dict:find(MFA, ActiveDict) of + try maps:find(MFA, ActiveMap) of error -> SpecData = {TypeSpec, Xtra}, - NewActiveDict = + NewActiveMap = dialyzer_contracts:store_tmp_contract(MFA, {File, Ln}, SpecData, - ActiveDict, RecordsDict), - {NewSpecDict, NewCallbackDict} = + ActiveMap, RecordsMap), + {NewSpecMap, NewCallbackMap} = case Contract of - spec -> {NewActiveDict, CallbackDict}; - callback -> {SpecDict, NewActiveDict} + spec -> {NewActiveMap, CallbackMap}; + callback -> {SpecMap, NewActiveMap} end, - get_spec_info(Left, NewSpecDict, NewCallbackDict, - RecordsDict, ModName, OptCb, File); + get_spec_info(Left, NewSpecMap, NewCallbackMap, + RecordsMap, ModName, OptCb, File); {ok, {{OtherFile, L}, _D}} -> {Mod, Fun, Arity} = MFA, Msg = flat_format(" Contract/callback for function ~w:~w/~w " @@ -502,16 +506,16 @@ get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left], [Ln, Error])} end; get_spec_info([{attribute, _, file, {IncludeFile, _}}|Left], - SpecDict, CallbackDict, RecordsDict, ModName, OptCb, _File) -> - get_spec_info(Left, SpecDict, CallbackDict, - RecordsDict, ModName, OptCb, IncludeFile); -get_spec_info([_Other|Left], SpecDict, CallbackDict, - RecordsDict, ModName, OptCb, File) -> - get_spec_info(Left, SpecDict, CallbackDict, - RecordsDict, ModName, OptCb, File); -get_spec_info([], SpecDict, CallbackDict, - _RecordsDict, _ModName, _OptCb, _File) -> - {ok, SpecDict, CallbackDict}. + SpecMap, CallbackMap, RecordsMap, ModName, OptCb, _File) -> + get_spec_info(Left, SpecMap, CallbackMap, + RecordsMap, ModName, OptCb, IncludeFile); +get_spec_info([_Other|Left], SpecMap, CallbackMap, + RecordsMap, ModName, OptCb, File) -> + get_spec_info(Left, SpecMap, CallbackMap, + RecordsMap, ModName, OptCb, File); +get_spec_info([], SpecMap, CallbackMap, + _RecordsMap, _ModName, _OptCb, _File) -> + {ok, SpecMap, CallbackMap}. -spec get_fun_meta_info(module(), abstract_code(), [dial_warn_tag()]) -> dialyzer_codeserver:fun_meta_info() | {'error', string()}. @@ -707,7 +711,7 @@ format_errors([]) -> -spec format_sig(erl_types:erl_type()) -> string(). format_sig(Type) -> - format_sig(Type, dict:new()). + format_sig(Type, maps:new()). -spec format_sig(erl_types:erl_type(), type_table()) -> string(). @@ -959,9 +963,7 @@ label(Tree) -> -spec parallelism() -> integer(). parallelism() -> - CPUs = erlang:system_info(logical_processors_available), - Schedulers = erlang:system_info(schedulers), - min(CPUs, Schedulers). + erlang:system_info(schedulers_online). -spec family([{K,V}]) -> [{K,[V]}]. diff --git a/lib/dialyzer/test/small_SUITE_data/src/anno.erl b/lib/dialyzer/test/small_SUITE_data/src/anno.erl new file mode 100644 index 0000000000..70f1d42141 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/anno.erl @@ -0,0 +1,18 @@ +-module(anno). + +%% OTP-14131 + +-export([t1/0, t2/0, t3/0]). + +t1() -> + A = erl_parse:anno_from_term({attribute, 1, module, my_test}), + compile:forms([A], []). + +t2() -> + A = erl_parse:new_anno({attribute, 1, module, my_test}), + compile:forms([A], []). + +t3() -> + A = erl_parse:new_anno({attribute, 1, module, my_test}), + T = erl_parse:anno_to_term(A), + {attribute, 1, module, my_test} = T. diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index f39c3286df..7edfbf65df 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -236,7 +236,8 @@ -export([t_is_identifier/1]). -endif. --export_type([erl_type/0, opaques/0, type_table/0, var_table/0, cache/0]). +-export_type([erl_type/0, opaques/0, type_table/0, mod_records/0, + var_table/0, cache/0]). %%-define(DEBUG, true). @@ -379,8 +380,9 @@ -type type_value() :: {{module(), {file:name(), erl_anno:line()}, erl_parse:abstract_type(), ArgNames :: [atom()]}, erl_type()}. --type type_table() :: dict:dict(record_key() | type_key(), - record_value() | type_value()). +-type type_table() :: #{record_key() | type_key() => + record_value() | type_value()}. +-type mod_records() :: dict:dict(module(), type_table()). -opaque var_table() :: #{atom() => erl_type()}. @@ -749,16 +751,16 @@ decorate_tuples_in_sets([], _L, _Opaques, Acc) -> -spec t_opaque_from_records(type_table()) -> [erl_type()]. -t_opaque_from_records(RecDict) -> - OpaqueRecDict = - dict:filter(fun(Key, _Value) -> +t_opaque_from_records(RecMap) -> + OpaqueRecMap = + maps:filter(fun(Key, _Value) -> case Key of {opaque, _Name, _Arity} -> true; _ -> false end - end, RecDict), - OpaqueTypeDict = - dict:map(fun({opaque, Name, _Arity}, + end, RecMap), + OpaqueTypeMap = + maps:map(fun({opaque, Name, _Arity}, {{Module, _FileLine, _Form, ArgNames}, _Type}) -> %% Args = args_to_types(ArgNames), %% List = lists:zip(ArgNames, Args), @@ -767,8 +769,8 @@ t_opaque_from_records(RecDict) -> Rep = t_any(), % not used for anything right now Args = [t_any() || _ <- ArgNames], t_opaque(Module, Name, Args, Rep) - end, OpaqueRecDict), - [OpaqueType || {_Key, OpaqueType} <- dict:to_list(OpaqueTypeDict)]. + end, OpaqueRecMap), + [OpaqueType || {_Key, OpaqueType} <- maps:to_list(OpaqueTypeMap)]. %% Decompose opaque instances of type arg2 to structured types, in arg1 %% XXX: Same as t_unopaque @@ -802,10 +804,6 @@ list_struct_from_opaque(Types, Opaques) -> [t_struct_from_opaque(Type, Opaques) || Type <- Types]. %%----------------------------------------------------------------------------- - --type mod_records() :: dict:dict(module(), type_table()). - -%%----------------------------------------------------------------------------- %% Unit type. Signals non termination. %% @@ -4184,7 +4182,7 @@ t_map(Fun, T) -> -spec t_to_string(erl_type()) -> string(). t_to_string(T) -> - t_to_string(T, dict:new()). + t_to_string(T, maps:new()). -spec t_to_string(erl_type(), type_table()) -> string(). @@ -5246,7 +5244,7 @@ t_form_to_string({type, _L, union, Args}) -> t_form_to_string({type, _L, Name, []} = T) -> try M = mod, - D0 = dict:new(), + D0 = maps:new(), MR = dict:from_list([{M, D0}]), Site = {type, {M,Name,0}}, V = var_table__new(), @@ -5310,8 +5308,8 @@ is_erl_type(_) -> false. -spec lookup_record(atom(), type_table()) -> 'error' | {'ok', [{atom(), parse_form(), erl_type()}]}. -lookup_record(Tag, RecDict) when is_atom(Tag) -> - case dict:find({record, Tag}, RecDict) of +lookup_record(Tag, Table) when is_atom(Tag) -> + case maps:find({record, Tag}, Table) of {ok, {_FileLine, [{_Arity, Fields}]}} -> {ok, Fields}; {ok, {_FileLine, List}} when is_list(List) -> @@ -5325,18 +5323,18 @@ lookup_record(Tag, RecDict) when is_atom(Tag) -> -spec lookup_record(atom(), arity(), type_table()) -> 'error' | {'ok', [{atom(), parse_form(), erl_type()}]}. -lookup_record(Tag, Arity, RecDict) when is_atom(Tag) -> - case dict:find({record, Tag}, RecDict) of +lookup_record(Tag, Arity, Table) when is_atom(Tag) -> + case maps:find({record, Tag}, Table) of {ok, {_FileLine, [{Arity, Fields}]}} -> {ok, Fields}; {ok, {_FileLine, OrdDict}} -> orddict:find(Arity, OrdDict); error -> error end. -spec lookup_type(_, _, _) -> {'type' | 'opaque', type_value()} | 'error'. -lookup_type(Name, Arity, RecDict) -> - case dict:find({type, Name, Arity}, RecDict) of +lookup_type(Name, Arity, Table) -> + case maps:find({type, Name, Arity}, Table) of error -> - case dict:find({opaque, Name, Arity}, RecDict) of + case maps:find({opaque, Name, Arity}, Table) of error -> error; {ok, Found} -> {opaque, Found} end; @@ -5346,8 +5344,8 @@ lookup_type(Name, Arity, RecDict) -> -spec type_is_defined('type' | 'opaque', atom(), arity(), type_table()) -> boolean(). -type_is_defined(TypeOrOpaque, Name, Arity, RecDict) -> - dict:is_key({TypeOrOpaque, Name, Arity}, RecDict). +type_is_defined(TypeOrOpaque, Name, Arity, Table) -> + maps:is_key({TypeOrOpaque, Name, Arity}, Table). cannot_have_opaque(Type, TypeName, TypeNames) -> t_is_none(Type) orelse is_recursive(TypeName, TypeNames). diff --git a/lib/os_mon/src/memsup.erl b/lib/os_mon/src/memsup.erl index 4729d090f8..0a9a883390 100644 --- a/lib/os_mon/src/memsup.erl +++ b/lib/os_mon/src/memsup.erl @@ -701,6 +701,7 @@ get_os_wordsize_with_uname() -> "sparc64" -> 64; "amd64" -> 64; "ppc64" -> 64; + "s390x" -> 64; _ -> 32 end. diff --git a/lib/runtime_tools/doc/src/LTTng.xml b/lib/runtime_tools/doc/src/LTTng.xml index 82a4c79379..7aae5e5c41 100644 --- a/lib/runtime_tools/doc/src/LTTng.xml +++ b/lib/runtime_tools/doc/src/LTTng.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="utf8" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> diff --git a/lib/snmp/src/app/snmp.appup.src b/lib/snmp/src/app/snmp.appup.src index ca61782639..db09ec3dc5 100644 --- a/lib/snmp/src/app/snmp.appup.src +++ b/lib/snmp/src/app/snmp.appup.src @@ -8,6 +8,10 @@ %% {update, snmpa_local_db, soft, soft_purge, soft_purge, []} %% {add_module, snmpm_net_if_mt} [ + {<<"5\\.2\\.4">>, + [{load_module, snmp, soft_purge, soft_purge, []}, + {load_module, snmpc_lib, soft_purge, soft_purge, []}, + {load_module, snmpc_mib_gram, soft_purge, soft_purge, []}]}, {<<"5\\..*">>, [{restart_application, snmp}]}, {<<"4\\..*">>, [{restart_application, snmp}]} ], @@ -17,6 +21,10 @@ %% {remove, {snmpm_net_if_mt, soft_purge, soft_purge}} [ + {<<"5\\.2\\.4">>, + [{load_module, snmp, soft_purge, soft_purge, []}, + {load_module, snmpc_lib, soft_purge, soft_purge, []}, + {load_module, snmpc_mib_gram, soft_purge, soft_purge, []}]}, {<<"5\\..*">>, [{restart_application, snmp}]}, {<<"4\\..*">>, [{restart_application, snmp}]} ] diff --git a/lib/snmp/src/app/snmp.erl b/lib/snmp/src/app/snmp.erl index df3933ea01..8a736f688b 100644 --- a/lib/snmp/src/app/snmp.erl +++ b/lib/snmp/src/app/snmp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -573,9 +573,16 @@ print_mod_info(Prefix, {Module, Info}) -> CompDate = case key1search(compile_time, Info) of {value, {Year, Month, Day, Hour, Min, Sec}} -> - lists:flatten( - io_lib:format("~w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", - [Year, Month, Day, Hour, Min, Sec])); + io_lib:format( + "~w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", + [Year, Month, Day, Hour, Min, Sec]); + _ -> + "Not found" + end, + Digest = + case key1search(md5, Info) of + {value, MD5} when is_binary(MD5) -> + [io_lib:format("~2.16.0b", [Byte]) || <<Byte>> <= MD5]; _ -> "Not found" end, @@ -583,12 +590,14 @@ print_mod_info(Prefix, {Module, Info}) -> "~s Vsn: ~s~n" "~s App vsn: ~s~n" "~s Compiler ver: ~s~n" - "~s Compile time: ~s~n", + "~s Compile time: ~s~n" + "~s MD5 digest: ~s~n", [Prefix, Module, Prefix, Vsn, Prefix, AppVsn, - Prefix, CompVer, - Prefix, CompDate]), + Prefix, CompVer, + Prefix, CompDate, + Prefix, Digest]), ok. key1search(Key, Vals) -> @@ -617,7 +626,7 @@ versions1() -> Error -> Error end. - + versions2() -> case ms2() of {ok, Mods} -> @@ -625,25 +634,56 @@ versions2() -> Error -> Error end. - + version_info(Mods) -> SysInfo = sys_info(), OsInfo = os_info(), ModInfo = [mod_version_info(Mod) || Mod <- Mods], [{sys_info, SysInfo}, {os_info, OsInfo}, {mod_info, ModInfo}]. - + mod_version_info(Mod) -> Info = Mod:module_info(), - {value, {attributes, Attr}} = lists:keysearch(attributes, 1, Info), - {value, {vsn, [Vsn]}} = lists:keysearch(vsn, 1, Attr), - {value, {app_vsn, AppVsn}} = lists:keysearch(app_vsn, 1, Attr), - {value, {compile, Comp}} = lists:keysearch(compile, 1, Info), - {value, {version, Ver}} = lists:keysearch(version, 1, Comp), - {value, {time, Time}} = lists:keysearch(time, 1, Comp), - {Mod, [{vsn, Vsn}, - {app_vsn, AppVsn}, - {compiler_version, Ver}, - {compile_time, Time}]}. + {Mod, + case key1search(attributes, Info) of + {value, Attr} -> + case key1search(vsn, Attr) of + {value, [Vsn]} -> + [{vsn, Vsn}]; + not_found -> + [] + end ++ + case key1search(app_vsn, Attr) of + {value, AppVsn} -> + [{app_vsn, AppVsn}]; + not_found -> + [] + end; + not_found -> + [] + end ++ + case key1search(compile, Info) of + {value, Comp} -> + case key1search(version, Comp) of + {value, Ver} -> + [{compiler_version, Ver}]; + not_found -> + [] + end ++ + case key1search(time, Comp) of + {value, Ver} -> + [{compile_time, Ver}]; + not_found -> + [] + end; + not_found -> + [] + end ++ + case key1search(md5, Info) of + {value, Bin} -> + [{md5, Bin}]; + not_found -> + [] + end}. sys_info() -> SysArch = string:strip(erlang:system_info(system_architecture),right,$\n), diff --git a/lib/snmp/src/compile/snmpc_lib.erl b/lib/snmp/src/compile/snmpc_lib.erl index 51690b6e7e..33ddd78308 100644 --- a/lib/snmp/src/compile/snmpc_lib.erl +++ b/lib/snmp/src/compile/snmpc_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -99,7 +99,7 @@ make_ASN1type({{type_with_size,Type,{range,Lo,Hi}},Line}) -> print_error("Undefined type '~w'",[Type],Line), guess_string_type() end; -make_ASN1type({{integer_with_enum,Type,Enums},Line}) -> +make_ASN1type({{type_with_enum,Type,Enums},Line}) -> case lookup_vartype(Type) of {value,ASN1type} -> ASN1type#asn1_type{assocList = [{enums, Enums}]}; false -> diff --git a/lib/snmp/src/compile/snmpc_mib_gram.yrl b/lib/snmp/src/compile/snmpc_mib_gram.yrl index 743c3a6550..14a668127e 100644 --- a/lib/snmp/src/compile/snmpc_mib_gram.yrl +++ b/lib/snmp/src/compile/snmpc_mib_gram.yrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -387,10 +387,12 @@ syntax -> type : {{type, cat('$1')},line_of('$1')}. syntax -> type size : {{type_with_size, cat('$1'), '$2'},line_of('$1')}. syntax -> usertype size : {{type_with_size,val('$1'), '$2'},line_of('$1')}. syntax -> 'INTEGER' '{' namedbits '}' : - {{integer_with_enum, 'INTEGER', '$3'}, line_of('$1')}. + {{type_with_enum, 'INTEGER', '$3'}, line_of('$1')}. syntax -> 'BITS' '{' namedbits '}' : ensure_ver(2,'$1'), {{bits, '$3'}, line_of('$1')}. +syntax -> usertype '{' namedbits '}' : + {{type_with_enum, 'INTEGER', '$3'}, line_of('$1')}. syntax -> 'SEQUENCE' 'OF' usertype : {{sequence_of,val('$3')},line_of('$1')}. diff --git a/lib/snmp/test/snmp_compiler_test.erl b/lib/snmp/test/snmp_compiler_test.erl index 2c8851c2a7..9b3c2bfd2c 100644 --- a/lib/snmp/test/snmp_compiler_test.erl +++ b/lib/snmp/test/snmp_compiler_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. +%% Copyright Ericsson AB 2003-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -56,7 +56,8 @@ otp_8574/1, otp_8595/1, otp_10799/1, - otp_10808/1 + otp_10808/1, + otp_14145/1 ]). @@ -135,7 +136,8 @@ all() -> ]. groups() -> - [{tickets, [], [otp_6150, otp_8574, otp_8595, otp_10799, otp_10808]}]. + [{tickets, [], + [otp_6150, otp_8574, otp_8595, otp_10799, otp_10808, otp_14145]}]. init_per_group(_GroupName, Config) -> Config. @@ -431,6 +433,30 @@ otp_10808(Config) when is_list(Config) -> %%====================================================================== +otp_14145(suite) -> + []; +otp_14145(Config) when is_list(Config) -> + put(tname, otp10808), + p("starting with Config: ~p~n", [Config]), + + Dir = ?config(case_top_dir, Config), + MibDir = ?config(mib_dir, Config), + MibName = "OTP14145-MIB", + MibFile = join(MibDir, MibName++".mib"), + ?line {ok, MibBin} = + snmpc:compile(MibFile, [{outdir, Dir}, + {verbosity, trace}, + {group_check, false}, + module_compliance]), + p("Mib: ~n~p~n", [MibBin]), + MIB = read_mib(MibBin), + Oid = [1,3,6,1,2,1,67,4], + check_mib(MIB#mib.mes, Oid, undefined), + ok. + + +%%====================================================================== + augments_extra_info(suite) -> []; augments_extra_info(Config) when is_list(Config) -> diff --git a/lib/snmp/test/snmp_test_data/OTP14145-MIB.mib b/lib/snmp/test/snmp_test_data/OTP14145-MIB.mib new file mode 100644 index 0000000000..f29c65c4c2 --- /dev/null +++ b/lib/snmp/test/snmp_test_data/OTP14145-MIB.mib @@ -0,0 +1,44 @@ +OTP14145-MIB DEFINITIONS ::= BEGIN + +IMPORTS + MODULE-IDENTITY, OBJECT-TYPE, + mib-2 FROM SNMPv2-SMI + InetAddressType, InetAddress FROM INET-ADDRESS-MIB + MODULE-COMPLIANCE, OBJECT-GROUP FROM SNMPv2-CONF; + +testMibId MODULE-IDENTITY + LAST-UPDATED "200608210000Z" -- 21 August 2006 + ORGANIZATION "a" + CONTACT-INFO "a" + DESCRIPTION "a" + REVISION "200608210000Z" -- 21 August 2006 + DESCRIPTION "a" + ::= { mib-2 67 } + +testObj OBJECT-TYPE + SYNTAX InetAddressType + -- SYNTAX InetAddress + MAX-ACCESS read-only + STATUS current + DESCRIPTION "a" + ::= { testMibId 2 } + +testObjId OBJECT IDENTIFIER ::= { testMibId 3 } + +testMibCompliance MODULE-COMPLIANCE + STATUS current + DESCRIPTION "a" + MODULE + OBJECT testObj + SYNTAX InetAddressType { ipv4(1), ipv6(2) } + -- SYNTAX InetAddress ( SIZE(4|16) ) + DESCRIPTION "a" + ::= { testMibId 4 } + +testObjGroup OBJECT-GROUP + OBJECTS { testObj } + STATUS current + DESCRIPTION "a" + ::= { testObjId 1 } + +END diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 28eba0d0d6..30b8ee1124 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2016. All Rights Reserved. +# Copyright Ericsson AB 1997-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.2.4 +SNMP_VSN = 5.2.5 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 7451c9e6d0..8718e92fa2 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -1206,7 +1206,7 @@ handle_event(info, {Proto, Sock, NewData}, StateName, D0 = #data{socket = Sock, catch _C:_E -> disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Encountered unexpected input"}, + description = "Bad packet"}, StateName, D) end; @@ -1221,13 +1221,12 @@ handle_event(info, {Proto, Sock, NewData}, StateName, D0 = #data{socket = Sock, {bad_mac, Ssh1} -> disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad mac"}, + description = "Bad packet"}, StateName, D0#data{ssh_params=Ssh1}); - {error, {exceeds_max_size,PacketLen}} -> + {error, {exceeds_max_size,_PacketLen}} -> disconnect(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Bad packet length " - ++ integer_to_list(PacketLen)}, + description = "Bad packet"}, StateName, D0) catch _C:_E -> diff --git a/lib/ssh/src/ssh_dbg.erl b/lib/ssh/src/ssh_dbg.erl index dff2bae9f2..0345bbdea7 100644 --- a/lib/ssh/src/ssh_dbg.erl +++ b/lib/ssh/src/ssh_dbg.erl @@ -50,50 +50,61 @@ messages(Write, MangleArg) when is_function(Write,2), is_function(MangleArg,1) -> catch dbg:start(), setup_tracer(Write, MangleArg), - dbg:p(new,c), + dbg:p(new,[c,timestamp]), dbg_ssh_messages(). dbg_ssh_messages() -> dbg:tp(ssh_message,encode,1, x), dbg:tp(ssh_message,decode,1, x), - dbg:tpl(ssh_transport,select_algorithm,3, x). - + dbg:tpl(ssh_transport,select_algorithm,3, x), + dbg:tp(ssh_transport,hello_version_msg,1, x), + dbg:tp(ssh_transport,handle_hello_version,1, x). + %%%---------------------------------------------------------------- stop() -> dbg:stop(). %%%================================================================ -msg_formater({trace,Pid,call,{ssh_message,encode,[Msg]}}, D) -> - fmt("~nSEND ~p ~s~n", [Pid,wr_record(shrink_bin(Msg))], D); -msg_formater({trace,_Pid,return_from,{ssh_message,encode,1},_Res}, D) -> +msg_formater({trace_ts,Pid,call,{ssh_message,encode,[Msg]},TS}, D) -> + fmt("~n~s SEND ~p ~s~n", [ts(TS),Pid,wr_record(shrink_bin(Msg))], D); +msg_formater({trace_ts,_Pid,return_from,{ssh_message,encode,1},_Res,_TS}, D) -> D; -msg_formater({trace,_Pid,call,{ssh_message,decode,_}}, D) -> +msg_formater({trace_ts,_Pid,call,{ssh_message,decode,_},_TS}, D) -> D; -msg_formater({trace,Pid,return_from,{ssh_message,decode,1},Msg}, D) -> - fmt("~n~p RECV ~s~n", [Pid,wr_record(shrink_bin(Msg))], D); +msg_formater({trace_ts,Pid,return_from,{ssh_message,decode,1},Msg,TS}, D) -> + fmt("~n~s ~p RECV ~s~n", [ts(TS),Pid,wr_record(shrink_bin(Msg))], D); -msg_formater({trace,_Pid,call,{ssh_transport,select_algorithm,_}}, D) -> +msg_formater({trace_ts,_Pid,call,{ssh_transport,select_algorithm,_},_TS}, D) -> + D; +msg_formater({trace_ts,Pid,return_from,{ssh_transport,select_algorithm,3},{ok,Alg},TS}, D) -> + fmt("~n~s ~p ALGORITHMS~n~s~n", [ts(TS),Pid, wr_record(Alg)], D); + +msg_formater({trace_ts,_Pid,call,{ssh_transport,hello_version_msg,_},_TS}, D) -> D; -msg_formater({trace,Pid,return_from,{ssh_transport,select_algorithm,3},{ok,Alg}}, D) -> - fmt("~n~p ALGORITHMS~n~s~n", [Pid, wr_record(Alg)], D); +msg_formater({trace_ts,Pid,return_from,{ssh_transport,hello_version_msg,1},Hello,TS}, D) -> + fmt("~n~s ~p TCP SEND HELLO~n ~p~n", [ts(TS),Pid,lists:flatten(Hello)], D); +msg_formater({trace_ts,Pid,call,{ssh_transport,handle_hello_version,[Hello]},TS}, D) -> + fmt("~n~s ~p RECV HELLO~n ~p~n", [ts(TS),Pid,lists:flatten(Hello)], D); +msg_formater({trace_ts,_Pid,return_from,{ssh_transport,handle_hello_version,1},_,_TS}, D) -> + D; -msg_formater({trace,Pid,send,{tcp,Sock,Bytes},Pid}, D) -> - fmt("~n~p TCP SEND on ~p~n ~p~n", [Pid,Sock, shrink_bin(Bytes)], D); +msg_formater({trace_ts,Pid,send,{tcp,Sock,Bytes},Pid,TS}, D) -> + fmt("~n~s ~p TCP SEND on ~p~n ~p~n", [ts(TS),Pid,Sock, shrink_bin(Bytes)], D); -msg_formater({trace,Pid,send,{tcp,Sock,Bytes},Dest}, D) -> - fmt("~n~p TCP SEND from ~p TO ~p~n ~p~n", [Pid,Sock,Dest, shrink_bin(Bytes)], D); +msg_formater({trace_ts,Pid,send,{tcp,Sock,Bytes},Dest,TS}, D) -> + fmt("~n~s ~p TCP SEND from ~p TO ~p~n ~p~n", [ts(TS),Pid,Sock,Dest, shrink_bin(Bytes)], D); -msg_formater({trace,Pid,send,ErlangMsg,Dest}, D) -> - fmt("~n~p ERL MSG SEND TO ~p~n ~p~n", [Pid,Dest, shrink_bin(ErlangMsg)], D); +msg_formater({trace_ts,Pid,send,ErlangMsg,Dest,TS}, D) -> + fmt("~n~s ~p ERL MSG SEND TO ~p~n ~p~n", [ts(TS),Pid,Dest, shrink_bin(ErlangMsg)], D); -msg_formater({trace,Pid,'receive',{tcp,Sock,Bytes}}, D) -> - fmt("~n~p TCP RECEIVE on ~p~n ~p~n", [Pid,Sock,shrink_bin(Bytes)], D); +msg_formater({trace_ts,Pid,'receive',{tcp,Sock,Bytes},TS}, D) -> + fmt("~n~s ~p TCP RECEIVE on ~p~n ~p~n", [ts(TS),Pid,Sock,shrink_bin(Bytes)], D); -msg_formater({trace,Pid,'receive',ErlangMsg}, D) -> - fmt("~n~p ERL MSG RECEIVE~n ~p~n", [Pid,shrink_bin(ErlangMsg)], D); +msg_formater({trace_ts,Pid,'receive',ErlangMsg,TS}, D) -> + fmt("~n~s ~p ERL MSG RECEIVE~n ~p~n", [ts(TS),Pid,shrink_bin(ErlangMsg)], D); msg_formater(M, D) -> @@ -106,6 +117,11 @@ msg_formater(M, D) -> fmt(Fmt, Args, D=#data{writer=Write,acc=Acc}) -> D#data{acc = Write(io_lib:format(Fmt, Args), Acc)}. +ts({_,_,Usec}=Now) -> + {_Date,{HH,MM,SS}} = calendar:now_to_local_time(Now), + io_lib:format("~.2.0w:~.2.0w:~.2.0w.~.6.0w",[HH,MM,SS,Usec]); +ts(_) -> + "-". %%%---------------------------------------------------------------- setup_tracer(Write, MangleArg) -> Handler = fun(Arg, D) -> @@ -116,11 +132,11 @@ setup_tracer(Write, MangleArg) -> ok. %%%---------------------------------------------------------------- -shrink_bin(B) when is_binary(B), size(B)>100 -> {'*** SHRINKED BIN', +shrink_bin(B) when is_binary(B), size(B)>256 -> {'*** SHRINKED BIN', size(B), - element(1,split_binary(B,20)), + element(1,split_binary(B,64)), '...', - element(2,split_binary(B,size(B)-20)) + element(2,split_binary(B,size(B)-64)) }; shrink_bin(L) when is_list(L) -> lists:map(fun shrink_bin/1, L); shrink_bin(T) when is_tuple(T) -> list_to_tuple(shrink_bin(tuple_to_list(T))); diff --git a/lib/ssh/src/ssh_sftpd_file_api.erl b/lib/ssh/src/ssh_sftpd_file_api.erl index 78f452df67..e444e52ac0 100644 --- a/lib/ssh/src/ssh_sftpd_file_api.erl +++ b/lib/ssh/src/ssh_sftpd_file_api.erl @@ -36,7 +36,7 @@ -callback list_dir(file:name(), State::term()) -> {{ok, Filenames::term()}, State::term()} | {{error, Reason::term()}, State::term()}. -callback make_dir(Dir::term(), State::term()) -> - {{ok, State::term()},State::term()} | {{error, Reason::term()}, State::term()}. + {ok, State::term()} | {{error, Reason::term()}, State::term()}. -callback make_symlink(Path2::term(), Path::term(), State::term()) -> {ok, State::term()} | {{error, Reason::term()}, State::term()}. -callback open(Path::term(), Flags::term(), State::term()) -> diff --git a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl index 0f8a838f97..8ca29b9399 100644 --- a/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl +++ b/lib/ssh/test/property_test/ssh_eqc_encode_decode.erl @@ -184,10 +184,7 @@ gen_byte(N) when N>0 -> [gen_byte() || _ <- lists:seq(1,N)]. gen_char() -> choose($a,$z). -gen_mpint() -> ?LET(Size, choose(1,20), - ?LET(Str, vector(Size, gen_byte()), - gen_string( strip_0s(Str) ) - )). +gen_mpint() -> ?LET(I, largeint(), ssh_bits:mpint(I)). strip_0s([0|T]) -> strip_0s(T); strip_0s(X) -> X. diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index edc7e0d8b2..916b41742e 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -424,6 +424,14 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid </taglist> </item> + + <tag><c>max_handshake_size</c></tag> + <item> + <p>Integer (24 bits unsigned). Used to limit the size of + valid TLS handshake packets to avoid DoS attacks. + Defaults to 256*1024.</p> + </item> + </taglist> </item> diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile index 3dda1a3316..2e7df9792e 100644 --- a/lib/ssl/src/Makefile +++ b/lib/ssl/src/Makefile @@ -48,9 +48,17 @@ MODULES= \ dtls \ ssl_alert \ ssl_app \ - ssl_dist_sup\ ssl_sup \ + ssl_admin_sup\ + tls_connection_sup \ + ssl_connection_sup \ + ssl_listen_tracker_sup\ + dtls_connection_sup \ + dtls_udp_listener\ dtls_udp_sup \ + ssl_dist_sup\ + ssl_dist_admin_sup\ + ssl_dist_connection_sup\ inet_tls_dist \ inet6_tls_dist \ ssl_certificate\ @@ -61,21 +69,18 @@ MODULES= \ dtls_connection \ ssl_config \ ssl_connection \ - tls_connection_sup \ - dtls_connection_sup \ tls_handshake \ dtls_handshake\ ssl_handshake\ ssl_manager \ ssl_session \ ssl_session_cache \ + ssl_pem_cache \ ssl_crl\ ssl_crl_cache \ ssl_crl_hash_dir \ tls_socket \ dtls_socket \ - dtls_udp_listener\ - ssl_listen_tracker_sup \ tls_record \ dtls_record \ ssl_record \ diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 9c5d795848..148989174d 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -10,12 +10,14 @@ tls_v1, ssl_v3, ssl_v2, + tls_connection_sup, %% DTLS dtls_connection, dtls_handshake, dtls_record, dtls_socket, dtls_v1, + dtls_connection_sup, dtls_udp_listener, dtls_udp_sup, %% API @@ -31,16 +33,19 @@ ssl_cipher, ssl_srp_primes, ssl_alert, - ssl_listen_tracker_sup, + ssl_listen_tracker_sup, %% may be used by DTLS over SCTP %% Erlang Distribution over SSL/TLS inet_tls_dist, inet6_tls_dist, ssl_tls_dist_proxy, ssl_dist_sup, - %% SSL/TLS session handling + ssl_dist_connection_sup, + ssl_dist_admin_sup, + %% SSL/TLS session and cert handling ssl_session, ssl_session_cache, ssl_manager, + ssl_pem_cache, ssl_pkix_db, ssl_certificate, %% CRL handling @@ -51,8 +56,8 @@ %% App structure ssl_app, ssl_sup, - tls_connection_sup, - dtls_connection_sup + ssl_admin_sup, + ssl_connection_sup ]}, {registered, [ssl_sup, ssl_manager]}, {applications, [crypto, public_key, kernel, stdlib]}, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index c72ee44a95..4a5a7e25ea 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -577,7 +577,7 @@ prf(#sslsocket{pid = {Listen,_}}, _,_,_,_) when is_port(Listen) -> %% Description: Clear the PEM cache %%-------------------------------------------------------------------- clear_pem_cache() -> - ssl_manager:clear_pem_cache(). + ssl_pem_cache:clear(). %%--------------------------------------------------------------- -spec format_error({error, term()}) -> list(). @@ -765,7 +765,8 @@ handle_options(Opts0, Role) -> client, Role), crl_check = handle_option(crl_check, Opts, false), crl_cache = handle_option(crl_cache, Opts, {ssl_crl_cache, {internal, []}}), - v2_hello_compatible = handle_option(v2_hello_compatible, Opts, false) + v2_hello_compatible = handle_option(v2_hello_compatible, Opts, false), + max_handshake_size = handle_option(max_handshake_size, Opts, ?DEFAULT_MAX_HANDSHAKE_SIZE) }, CbInfo = proplists:get_value(cb_info, Opts, default_cb_info(Protocol)), @@ -780,7 +781,8 @@ handle_options(Opts0, Role) -> alpn_preferred_protocols, next_protocols_advertised, client_preferred_next_protocols, log_alert, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, - fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, v2_hello_compatible], + fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, v2_hello_compatible, + max_handshake_size], SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) @@ -1028,6 +1030,8 @@ validate_option(beast_mitigation, Value) when Value == one_n_minus_one orelse Value; validate_option(v2_hello_compatible, Value) when is_boolean(Value) -> Value; +validate_option(max_handshake_size, Value) when is_integer(Value) andalso Value =< ?MAX_UNIT24 -> + Value; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). diff --git a/lib/ssl/src/ssl_admin_sup.erl b/lib/ssl/src/ssl_admin_sup.erl new file mode 100644 index 0000000000..9c96435753 --- /dev/null +++ b/lib/ssl/src/ssl_admin_sup.erl @@ -0,0 +1,95 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_admin_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0, manager_opts/0]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= + +init([]) -> + PEMCache = pem_cache_child_spec(), + SessionCertManager = session_and_cert_manager_child_spec(), + {ok, {{rest_for_one, 10, 3600}, [PEMCache, SessionCertManager]}}. + +manager_opts() -> + CbOpts = case application:get_env(ssl, session_cb) of + {ok, Cb} when is_atom(Cb) -> + InitArgs = session_cb_init_args(), + [{session_cb, Cb}, {session_cb_init_args, InitArgs}]; + _ -> + [] + end, + case application:get_env(ssl, session_lifetime) of + {ok, Time} when is_integer(Time) -> + [{session_lifetime, Time}| CbOpts]; + _ -> + CbOpts + end. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +pem_cache_child_spec() -> + Name = ssl_pem_cache, + StartFunc = {ssl_pem_cache, start_link, [[]]}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_pem_cache], + Type = worker, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +session_and_cert_manager_child_spec() -> + Opts = manager_opts(), + Name = ssl_manager, + StartFunc = {ssl_manager, start_link, [Opts]}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_manager], + Type = worker, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +session_cb_init_args() -> + case application:get_env(ssl, session_cb_init_args) of + {ok, Args} when is_list(Args) -> + Args; + _ -> + [] + end. diff --git a/lib/ssl/src/ssl_config.erl b/lib/ssl/src/ssl_config.erl index 0652d029c3..54f83928ee 100644 --- a/lib/ssl/src/ssl_config.erl +++ b/lib/ssl/src/ssl_config.erl @@ -41,9 +41,11 @@ init(SslOpts, Role) -> {ok, CertDbRef, CertDbHandle, FileRefHandle, CacheHandle, CRLDbHandle, OwnCert, PrivateKey, DHParams}. init_manager_name(false) -> - put(ssl_manager, ssl_manager:manager_name(normal)); + put(ssl_manager, ssl_manager:name(normal)), + put(ssl_cache, ssl_pem_cache:name(normal)); init_manager_name(true) -> - put(ssl_manager, ssl_manager:manager_name(dist)). + put(ssl_manager, ssl_manager:name(dist)), + put(ssl_cache, ssl_pem_cache:name(dist)). init_certificates(#ssl_options{cacerts = CaCerts, cacertfile = CACertFile, @@ -135,6 +137,8 @@ file_error(File, Throw) -> case Throw of {Opt,{badmatch, {error, {badmatch, Error}}}} -> throw({options, {Opt, binary_to_list(File), Error}}); + {Opt, {badmatch, Error}} -> + throw({options, {Opt, binary_to_list(File), Error}}); _ -> throw(Throw) end. diff --git a/lib/ssl/src/ssl_connection_sup.erl b/lib/ssl/src/ssl_connection_sup.erl new file mode 100644 index 0000000000..1a1f43e683 --- /dev/null +++ b/lib/ssl/src/ssl_connection_sup.erl @@ -0,0 +1,101 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_connection_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= + +init([]) -> + + TLSConnetionManager = tls_connection_manager_child_spec(), + %% Handles emulated options so that they inherited by the accept + %% socket, even when setopts is performed on the listen socket + ListenOptionsTracker = listen_options_tracker_child_spec(), + + DTLSConnetionManager = dtls_connection_manager_child_spec(), + DTLSUdpListeners = dtls_udp_listeners_spec(), + + {ok, {{one_for_one, 10, 3600}, [TLSConnetionManager, + ListenOptionsTracker, + DTLSConnetionManager, + DTLSUdpListeners + ]}}. + + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +tls_connection_manager_child_spec() -> + Name = tls_connection, + StartFunc = {tls_connection_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_connection_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +dtls_connection_manager_child_spec() -> + Name = dtls_connection, + StartFunc = {dtls_connection_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [dtls_connection_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +listen_options_tracker_child_spec() -> + Name = tls_socket, + StartFunc = {ssl_listen_tracker_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_socket], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +dtls_udp_listeners_spec() -> + Name = dtls_udp_listener, + StartFunc = {dtls_udp_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/ssl_dist_admin_sup.erl b/lib/ssl/src/ssl_dist_admin_sup.erl new file mode 100644 index 0000000000..f60806c4cb --- /dev/null +++ b/lib/ssl/src/ssl_dist_admin_sup.erl @@ -0,0 +1,74 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2016-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_dist_admin_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= + +init([]) -> + PEMCache = pem_cache_child_spec(), + SessionCertManager = session_and_cert_manager_child_spec(), + {ok, {{rest_for_one, 10, 3600}, [PEMCache, SessionCertManager]}}. + + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +pem_cache_child_spec() -> + Name = ssl_pem_cache_dist, + StartFunc = {ssl_pem_cache, start_link_dist, [[]]}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_pem_cache], + Type = worker, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +session_and_cert_manager_child_spec() -> + Opts = ssl_admin_sup:manager_opts(), + Name = ssl_dist_manager, + StartFunc = {ssl_manager, start_link_dist, [Opts]}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_manager], + Type = worker, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + diff --git a/lib/ssl/src/ssl_dist_connection_sup.erl b/lib/ssl/src/ssl_dist_connection_sup.erl new file mode 100644 index 0000000000..e5842c866e --- /dev/null +++ b/lib/ssl/src/ssl_dist_connection_sup.erl @@ -0,0 +1,79 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_dist_connection_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= + +init([]) -> + + TLSConnetionManager = tls_connection_manager_child_spec(), + %% Handles emulated options so that they inherited by the accept + %% socket, even when setopts is performed on the listen socket + ListenOptionsTracker = listen_options_tracker_child_spec(), + + {ok, {{one_for_one, 10, 3600}, [TLSConnetionManager, + ListenOptionsTracker + ]}}. + + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +tls_connection_manager_child_spec() -> + Name = dist_tls_connection, + StartFunc = {tls_connection_sup, start_link_dist, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_connection_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +listen_options_tracker_child_spec() -> + Name = dist_tls_socket, + StartFunc = {ssl_listen_tracker_sup, start_link_dist, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [tls_socket], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + diff --git a/lib/ssl/src/ssl_dist_sup.erl b/lib/ssl/src/ssl_dist_sup.erl index d47cd76bf5..690b896919 100644 --- a/lib/ssl/src/ssl_dist_sup.erl +++ b/lib/ssl/src/ssl_dist_sup.erl @@ -44,34 +44,29 @@ start_link() -> %%%========================================================================= init([]) -> - SessionCertManager = session_and_cert_manager_child_spec(), - ConnetionManager = connection_manager_child_spec(), - ListenOptionsTracker = listen_options_tracker_child_spec(), + AdminSup = ssl_admin_child_spec(), + ConnectionSup = ssl_connection_sup(), ProxyServer = proxy_server_child_spec(), - - {ok, {{one_for_all, 10, 3600}, [SessionCertManager, ConnetionManager, - ListenOptionsTracker, - ProxyServer]}}. + {ok, {{one_for_all, 10, 3600}, [AdminSup, ProxyServer, ConnectionSup]}}. %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -session_and_cert_manager_child_spec() -> - Opts = ssl_sup:manager_opts(), - Name = ssl_manager_dist, - StartFunc = {ssl_manager, start_link_dist, [Opts]}, +ssl_admin_child_spec() -> + Name = ssl_dist_admin_sup, + StartFunc = {ssl_dist_admin_sup, start_link , []}, Restart = permanent, Shutdown = 4000, - Modules = [ssl_manager], - Type = worker, + Modules = [ssl_admin_sup], + Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -connection_manager_child_spec() -> - Name = ssl_connection_dist, - StartFunc = {tls_connection_sup, start_link_dist, []}, - Restart = permanent, - Shutdown = infinity, - Modules = [tls_connection_sup], +ssl_connection_sup() -> + Name = ssl_dist_connection_sup, + StartFunc = {ssl_dist_connection_sup, start_link, []}, + Restart = permanent, + Shutdown = 4000, + Modules = [ssl_connection_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. @@ -83,12 +78,3 @@ proxy_server_child_spec() -> Modules = [ssl_tls_dist_proxy], Type = worker, {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -listen_options_tracker_child_spec() -> - Name = tls_socket_dist, - StartFunc = {ssl_listen_tracker_sup, start_link_dist, []}, - Restart = permanent, - Shutdown = 4000, - Modules = [tls_socket], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index fde92035a2..324b7dbde3 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -80,6 +80,9 @@ -define(CLIENT_KEY_EXCHANGE, 16). -define(FINISHED, 20). +-define(MAX_UNIT24, 8388607). +-define(DEFAULT_MAX_HANDSHAKE_SIZE, (256*1024)). + -record(random, { gmt_unix_time, % uint32 random_bytes % opaque random_bytes[28] diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 98b89bb811..c34af9f82c 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -142,7 +142,8 @@ signature_algs, eccs, honor_ecc_order :: boolean(), - v2_hello_compatible :: boolean() + v2_hello_compatible :: boolean(), + max_handshake_size :: integer() }). -record(socket_options, diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index 5bd9521de7..29b15f843f 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -32,10 +32,9 @@ new_session_id/1, clean_cert_db/2, register_session/2, register_session/3, invalidate_session/2, insert_crls/2, insert_crls/3, delete_crls/1, delete_crls/2, - invalidate_session/3, invalidate_pem/1, clear_pem_cache/0, manager_name/1]). + invalidate_session/3, name/1]). -% Spawn export --export([init_session_validator/1, init_pem_cache_validator/1]). +-export([init_session_validator/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -52,9 +51,7 @@ session_lifetime :: integer(), certificate_db :: db_handle(), session_validation_timer :: reference(), - last_delay_timer = {undefined, undefined},%% Keep for testing purposes - last_pem_check :: erlang:timestamp(), - clear_pem_cache :: integer(), + last_delay_timer = {undefined, undefined},%% Keep for testing purposes session_cache_client_max :: integer(), session_cache_server_max :: integer(), session_server_invalidator :: undefined | pid(), @@ -63,7 +60,6 @@ -define(GEN_UNIQUE_ID_MAX_TRIES, 10). -define(SESSION_VALIDATION_INTERVAL, 60000). --define(CLEAR_PEM_CACHE, 120000). -define(CLEAN_SESSION_DB, 60000). -define(CLEAN_CERT_DB, 500). -define(DEFAULT_MAX_SESSION_CACHE, 1000). @@ -74,14 +70,14 @@ %%==================================================================== %%-------------------------------------------------------------------- --spec manager_name(normal | dist) -> atom(). +-spec name(normal | dist) -> atom(). %% %% Description: Returns the registered name of the ssl manager process %% in the operation modes 'normal' and 'dist'. %%-------------------------------------------------------------------- -manager_name(normal) -> +name(normal) -> ?MODULE; -manager_name(dist) -> +name(dist) -> list_to_atom(atom_to_list(?MODULE) ++ "dist"). %%-------------------------------------------------------------------- @@ -91,9 +87,10 @@ manager_name(dist) -> %% and certificate caching. %%-------------------------------------------------------------------- start_link(Opts) -> - DistMangerName = manager_name(normal), - gen_server:start_link({local, DistMangerName}, - ?MODULE, [DistMangerName, Opts], []). + MangerName = name(normal), + CacheName = ssl_pem_cache:name(normal), + gen_server:start_link({local, MangerName}, + ?MODULE, [MangerName, CacheName, Opts], []). %%-------------------------------------------------------------------- -spec start_link_dist(list()) -> {ok, pid()} | ignore | {error, term()}. @@ -102,9 +99,10 @@ start_link(Opts) -> %% be used by the erlang distribution. Note disables soft upgrade! %%-------------------------------------------------------------------- start_link_dist(Opts) -> - DistMangerName = manager_name(dist), + DistMangerName = name(dist), + DistCacheName = ssl_pem_cache:name(dist), gen_server:start_link({local, DistMangerName}, - ?MODULE, [DistMangerName, Opts], []). + ?MODULE, [DistMangerName, DistCacheName, Opts], []). %%-------------------------------------------------------------------- -spec connection_init(binary()| {der, list()}, client | server, @@ -115,25 +113,10 @@ start_link_dist(Opts) -> %% Description: Do necessary initializations for a new connection. %%-------------------------------------------------------------------- connection_init({der, _} = Trustedcerts, Role, CRLCache) -> - case bypass_pem_cache() of - true -> - {ok, Extracted} = ssl_pkix_db:extract_trusted_certs(Trustedcerts), - call({connection_init, Extracted, Role, CRLCache}); - false -> - call({connection_init, Trustedcerts, Role, CRLCache}) - end; - -connection_init(<<>> = Trustedcerts, Role, CRLCache) -> - call({connection_init, Trustedcerts, Role, CRLCache}); - + {ok, Extracted} = ssl_pkix_db:extract_trusted_certs(Trustedcerts), + call({connection_init, Extracted, Role, CRLCache}); connection_init(Trustedcerts, Role, CRLCache) -> - case bypass_pem_cache() of - true -> - {ok, Extracted} = ssl_pkix_db:extract_trusted_certs(Trustedcerts), - call({connection_init, Extracted, Role, CRLCache}); - false -> - call({connection_init, Trustedcerts, Role, CRLCache}) - end. + call({connection_init, Trustedcerts, Role, CRLCache}). %%-------------------------------------------------------------------- -spec cache_pem_file(binary(), term()) -> {ok, term()} | {error, reason()}. @@ -141,31 +124,14 @@ connection_init(Trustedcerts, Role, CRLCache) -> %% Description: Cache a pem file and return its content. %%-------------------------------------------------------------------- cache_pem_file(File, DbHandle) -> - case bypass_pem_cache() of - true -> - ssl_pkix_db:decode_pem_file(File); - false -> - case ssl_pkix_db:lookup_cached_pem(DbHandle, File) of - [{Content,_}] -> - {ok, Content}; - [Content] -> - {ok, Content}; - undefined -> - call({cache_pem, File}) - end + case ssl_pkix_db:lookup(File, DbHandle) of + [Content] -> + {ok, Content}; + undefined -> + ssl_pem_cache:insert(File) end. %%-------------------------------------------------------------------- --spec clear_pem_cache() -> ok. -%% -%% Description: Clear the PEM cache -%%-------------------------------------------------------------------- -clear_pem_cache() -> - %% Not supported for distribution at the moement, should it be? - put(ssl_manager, manager_name(normal)), - call(unconditionally_clear_pem_cache). - -%%-------------------------------------------------------------------- -spec lookup_trusted_cert(term(), reference(), serialnumber(), issuer()) -> undefined | {ok, {der_cert(), #'OTPCertificate'{}}}. @@ -222,26 +188,22 @@ invalidate_session(Port, Session) -> load_mitigation(), cast({invalidate_session, Port, Session}). --spec invalidate_pem(File::binary()) -> ok. -invalidate_pem(File) -> - cast({invalidate_pem, File}). - insert_crls(Path, CRLs)-> insert_crls(Path, CRLs, normal). insert_crls(?NO_DIST_POINT_PATH = Path, CRLs, ManagerType)-> - put(ssl_manager, manager_name(ManagerType)), + put(ssl_manager, name(ManagerType)), cast({insert_crls, Path, CRLs}); insert_crls(Path, CRLs, ManagerType)-> - put(ssl_manager, manager_name(ManagerType)), + put(ssl_manager, name(ManagerType)), call({insert_crls, Path, CRLs}). delete_crls(Path)-> delete_crls(Path, normal). delete_crls(?NO_DIST_POINT_PATH = Path, ManagerType)-> - put(ssl_manager, manager_name(ManagerType)), + put(ssl_manager, name(ManagerType)), cast({delete_crls, Path}); delete_crls(Path, ManagerType)-> - put(ssl_manager, manager_name(ManagerType)), + put(ssl_manager, name(ManagerType)), call({delete_crls, Path}). %%==================================================================== @@ -255,8 +217,9 @@ delete_crls(Path, ManagerType)-> %% %% Description: Initiates the server %%-------------------------------------------------------------------- -init([Name, Opts]) -> - put(ssl_manager, Name), +init([ManagerName, PemCacheName, Opts]) -> + put(ssl_manager, ManagerName), + put(ssl_pem_cache, PemCacheName), process_flag(trap_exit, true), CacheCb = proplists:get_value(session_cb, Opts, ssl_session_cache), SessionLifeTime = @@ -270,16 +233,12 @@ init([Name, Opts]) -> proplists:get_value(session_cb_init_args, Opts, [])]), Timer = erlang:send_after(SessionLifeTime * 1000 + 5000, self(), validate_sessions), - Interval = pem_check_interval(), - erlang:send_after(Interval, self(), clear_pem_cache), {ok, #state{certificate_db = CertDb, session_cache_client = ClientSessionCache, session_cache_server = ServerSessionCache, session_cache_cb = CacheCb, session_lifetime = SessionLifeTime, session_validation_timer = Timer, - last_pem_check = os:timestamp(), - clear_pem_cache = Interval, session_cache_client_max = max_session_cache_size(session_cache_client_max), session_cache_server_max = @@ -330,21 +289,7 @@ handle_call({{new_session_id, Port}, _}, _, #state{session_cache_cb = CacheCb, session_cache_server = Cache} = State) -> Id = new_id(Port, ?GEN_UNIQUE_ID_MAX_TRIES, Cache, CacheCb), - {reply, Id, State}; - -handle_call({{cache_pem,File}, _Pid}, _, - #state{certificate_db = Db} = State) -> - try ssl_pkix_db:cache_pem_file(File, Db) of - Result -> - {reply, Result, State} - catch - _:Reason -> - {reply, {error, Reason}, State} - end; -handle_call({unconditionally_clear_pem_cache, _},_, - #state{certificate_db = [_,_,PemChace | _]} = State) -> - ssl_pkix_db:clear(PemChace), - {reply, ok, State}. + {reply, Id, State}. %%-------------------------------------------------------------------- -spec handle_cast(msg(), #state{}) -> {noreply, #state{}}. @@ -382,11 +327,6 @@ handle_cast({insert_crls, Path, CRLs}, handle_cast({delete_crls, CRLsOrPath}, #state{certificate_db = Db} = State) -> ssl_pkix_db:remove_crls(Db, CRLsOrPath), - {noreply, State}; - -handle_cast({invalidate_pem, File}, - #state{certificate_db = [_, _, PemCache | _]} = State) -> - ssl_pkix_db:remove(File, PemCache), {noreply, State}. %%-------------------------------------------------------------------- @@ -418,22 +358,14 @@ handle_info({delayed_clean_session, Key, Cache}, #state{session_cache_cb = Cache CacheCb:delete(Cache, Key), {noreply, State}; -handle_info(clear_pem_cache, #state{certificate_db = [_,_,PemChace | _], - clear_pem_cache = Interval, - last_pem_check = CheckPoint} = State) -> - NewCheckPoint = os:timestamp(), - start_pem_cache_validator(PemChace, CheckPoint), - erlang:send_after(Interval, self(), clear_pem_cache), - {noreply, State#state{last_pem_check = NewCheckPoint}}; - handle_info({clean_cert_db, Ref, File}, - #state{certificate_db = [CertDb,RefDb, PemCache | _]} = State) -> + #state{certificate_db = [CertDb, {RefDb, FileMapDb} | _]} = State) -> case ssl_pkix_db:lookup(Ref, RefDb) of undefined -> %% Alredy cleaned ok; _ -> - clean_cert_db(Ref, CertDb, RefDb, PemCache, File) + clean_cert_db(Ref, CertDb, RefDb, FileMapDb, File) end, {noreply, State}; @@ -523,14 +455,6 @@ delay_time() -> ?CLEAN_SESSION_DB end. -bypass_pem_cache() -> - case application:get_env(ssl, bypass_pem_cache) of - {ok, Bool} when is_boolean(Bool) -> - Bool; - _ -> - false - end. - max_session_cache_size(CacheType) -> case application:get_env(ssl, CacheType) of {ok, Size} when is_integer(Size) -> @@ -594,16 +518,11 @@ new_id(Port, Tries, Cache, CacheCb) -> new_id(Port, Tries - 1, Cache, CacheCb) end. -clean_cert_db(Ref, CertDb, RefDb, PemCache, File) -> +clean_cert_db(Ref, CertDb, RefDb, FileMapDb, File) -> case ssl_pkix_db:ref_count(Ref, RefDb, 0) of 0 -> - case ssl_pkix_db:lookup_cached_pem(PemCache, File) of - [{Content, Ref}] -> - ssl_pkix_db:insert(File, Content, PemCache); - _ -> - ok - end, ssl_pkix_db:remove(Ref, RefDb), + ssl_pkix_db:remove(File, FileMapDb), ssl_pkix_db:remove_trusted_certs(Ref, CertDb); _ -> ok @@ -687,42 +606,6 @@ exists_equivalent(#session{ exists_equivalent(Session, [ _ | Rest]) -> exists_equivalent(Session, Rest). -start_pem_cache_validator(PemCache, CheckPoint) -> - spawn_link(?MODULE, init_pem_cache_validator, - [[get(ssl_manager), PemCache, CheckPoint]]). - -init_pem_cache_validator([SslManagerName, PemCache, CheckPoint]) -> - put(ssl_manager, SslManagerName), - ssl_pkix_db:foldl(fun pem_cache_validate/2, - CheckPoint, PemCache). - -pem_cache_validate({File, _}, CheckPoint) -> - case file:read_file_info(File, []) of - {ok, #file_info{mtime = Time}} -> - case is_before_checkpoint(Time, CheckPoint) of - true -> - ok; - false -> - invalidate_pem(File) - end; - _ -> - invalidate_pem(File) - end, - CheckPoint. - -pem_check_interval() -> - case application:get_env(ssl, ssl_pem_cache_clean) of - {ok, Interval} when is_integer(Interval) -> - Interval; - _ -> - ?CLEAR_PEM_CACHE - end. - -is_before_checkpoint(Time, CheckPoint) -> - calendar:datetime_to_gregorian_seconds( - calendar:now_to_datetime(CheckPoint)) - - calendar:datetime_to_gregorian_seconds(Time) > 0. - add_trusted_certs(Pid, Trustedcerts, Db) -> try ssl_pkix_db:add_trusted_certs(Pid, Trustedcerts, Db) diff --git a/lib/ssl/src/ssl_pem_cache.erl b/lib/ssl/src/ssl_pem_cache.erl new file mode 100644 index 0000000000..2b31374bcc --- /dev/null +++ b/lib/ssl/src/ssl_pem_cache.erl @@ -0,0 +1,266 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 20016-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%---------------------------------------------------------------------- +%% Purpose: Manages ssl sessions and trusted certifacates +%%---------------------------------------------------------------------- + +-module(ssl_pem_cache). +-behaviour(gen_server). + +%% Internal application API +-export([start_link/1, + start_link_dist/1, + name/1, + insert/1, + clear/0]). + +% Spawn export +-export([init_pem_cache_validator/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-include("ssl_handshake.hrl"). +-include("ssl_internal.hrl"). +-include_lib("kernel/include/file.hrl"). + +-record(state, { + pem_cache, + last_pem_check :: erlang:timestamp(), + clear :: integer() + }). + +-define(CLEAR_PEM_CACHE, 120000). +-define(DEFAULT_MAX_SESSION_CACHE, 1000). + +%%==================================================================== +%% API +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec name(normal | dist) -> atom(). +%% +%% Description: Returns the registered name of the ssl cache process +%% in the operation modes 'normal' and 'dist'. +%%-------------------------------------------------------------------- +name(normal) -> + ?MODULE; +name(dist) -> + list_to_atom(atom_to_list(?MODULE) ++ "dist"). + +%%-------------------------------------------------------------------- +-spec start_link(list()) -> {ok, pid()} | ignore | {error, term()}. +%% +%% Description: Starts the ssl pem cache handler +%%-------------------------------------------------------------------- +start_link(_) -> + CacheName = name(normal), + gen_server:start_link({local, CacheName}, + ?MODULE, [CacheName], []). + +%%-------------------------------------------------------------------- +-spec start_link_dist(list()) -> {ok, pid()} | ignore | {error, term()}. +%% +%% Description: Starts a special instance of the ssl manager to +%% be used by the erlang distribution. Note disables soft upgrade! +%%-------------------------------------------------------------------- +start_link_dist(_) -> + DistCacheName = name(dist), + gen_server:start_link({local, DistCacheName}, + ?MODULE, [DistCacheName], []). + + +%%-------------------------------------------------------------------- +-spec insert(binary()) -> {ok, term()} | {error, reason()}. +%% +%% Description: Cache a pem file and return its content. +%%-------------------------------------------------------------------- +insert(File) -> + {ok, PemBin} = file:read_file(File), + Content = public_key:pem_decode(PemBin), + case bypass_cache() of + true -> + {ok, Content}; + false -> + cast({cache_pem, File, Content}), + {ok, Content} + end. + +%%-------------------------------------------------------------------- +-spec clear() -> ok. +%% +%% Description: Clear the PEM cache +%%-------------------------------------------------------------------- +clear() -> + %% Not supported for distribution at the moement, should it be? + put(ssl_pem_cache, name(normal)), + call(unconditionally_clear_pem_cache). + +-spec invalidate_pem(File::binary()) -> ok. +invalidate_pem(File) -> + cast({invalidate_pem, File}). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec init(list()) -> {ok, #state{}}. +%% Possible return values not used now. +%% | {ok, #state{}, timeout()} | ignore | {stop, term()}. +%% +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init([Name]) -> + put(ssl_pem_cache, Name), + process_flag(trap_exit, true), + PemCache = ssl_pkix_db:create_pem_cache(), + Interval = pem_check_interval(), + erlang:send_after(Interval, self(), clear_pem_cache), + {ok, #state{pem_cache = PemCache, + last_pem_check = os:timestamp(), + clear = Interval + }}. + +%%-------------------------------------------------------------------- +-spec handle_call(msg(), from(), #state{}) -> {reply, reply(), #state{}}. +%% Possible return values not used now. +%% {reply, reply(), #state{}, timeout()} | +%% {noreply, #state{}} | +%% {noreply, #state{}, timeout()} | +%% {stop, reason(), reply(), #state{}} | +%% {stop, reason(), #state{}}. +%% +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call({unconditionally_clear_pem_cache, _},_, + #state{pem_cache = PemCache} = State) -> + ssl_pkix_db:clear(PemCache), + {reply, ok, State}. + +%%-------------------------------------------------------------------- +-spec handle_cast(msg(), #state{}) -> {noreply, #state{}}. +%% Possible return values not used now. +%% | {noreply, #state{}, timeout()} | +%% {stop, reason(), #state{}}. +%% +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast({cache_pem, File, Content}, #state{pem_cache = Db} = State) -> + ssl_pkix_db:insert(File, Content, Db), + {noreply, State}; + +handle_cast({invalidate_pem, File}, #state{pem_cache = Db} = State) -> + ssl_pkix_db:remove(File, Db), + {noreply, State}. + + +%%-------------------------------------------------------------------- +-spec handle_info(msg(), #state{}) -> {noreply, #state{}}. +%% Possible return values not used now. +%% |{noreply, #state{}, timeout()} | +%% {stop, reason(), #state{}}. +%% +%% Description: Handling all non call/cast messages +%%------------------------------------------------------------------- +handle_info(clear_pem_cache, #state{pem_cache = PemCache, + clear = Interval, + last_pem_check = CheckPoint} = State) -> + NewCheckPoint = os:timestamp(), + start_pem_cache_validator(PemCache, CheckPoint), + erlang:send_after(Interval, self(), clear_pem_cache), + {noreply, State#state{last_pem_check = NewCheckPoint}}; + +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +-spec terminate(reason(), #state{}) -> ok. +%% +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(_Reason, #state{}) -> + ok. + +%%-------------------------------------------------------------------- +-spec code_change(term(), #state{}, list()) -> {ok, #state{}}. +%% +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +call(Msg) -> + gen_server:call(get(ssl_pem_cache), {Msg, self()}, infinity). + +cast(Msg) -> + gen_server:cast(get(ssl_pem_cache), Msg). + +start_pem_cache_validator(PemCache, CheckPoint) -> + spawn_link(?MODULE, init_pem_cache_validator, + [[get(ssl_pem_cache), PemCache, CheckPoint]]). + +init_pem_cache_validator([CacheName, PemCache, CheckPoint]) -> + put(ssl_pem_cache, CacheName), + ssl_pkix_db:foldl(fun pem_cache_validate/2, + CheckPoint, PemCache). + +pem_cache_validate({File, _}, CheckPoint) -> + case file:read_file_info(File, []) of + {ok, #file_info{mtime = Time}} -> + case is_before_checkpoint(Time, CheckPoint) of + true -> + ok; + false -> + invalidate_pem(File) + end; + _ -> + invalidate_pem(File) + end, + CheckPoint. + +is_before_checkpoint(Time, CheckPoint) -> + calendar:datetime_to_gregorian_seconds( + calendar:now_to_datetime(CheckPoint)) - + calendar:datetime_to_gregorian_seconds(Time) > 0. + +pem_check_interval() -> + case application:get_env(ssl, ssl_pem_cache_clean) of + {ok, Interval} when is_integer(Interval) -> + Interval; + _ -> + ?CLEAR_PEM_CACHE + end. + +bypass_cache() -> + case application:get_env(ssl, bypass_pem_cache) of + {ok, Bool} when is_boolean(Bool) -> + Bool; + _ -> + false + end. diff --git a/lib/ssl/src/ssl_pkix_db.erl b/lib/ssl/src/ssl_pkix_db.erl index b4299969e4..961a555873 100644 --- a/lib/ssl/src/ssl_pkix_db.erl +++ b/lib/ssl/src/ssl_pkix_db.erl @@ -28,11 +28,11 @@ -include_lib("public_key/include/public_key.hrl"). -include_lib("kernel/include/file.hrl"). --export([create/0, add_crls/3, remove_crls/2, remove/1, add_trusted_certs/3, +-export([create/0, create_pem_cache/0, + add_crls/3, remove_crls/2, remove/1, add_trusted_certs/3, extract_trusted_certs/1, remove_trusted_certs/2, insert/3, remove/2, clear/1, db_size/1, ref_count/3, lookup_trusted_cert/4, foldl/3, select_cert_by_issuer/2, - lookup_cached_pem/2, cache_pem_file/2, cache_pem_file/3, decode_pem_file/1, lookup/2]). %%==================================================================== @@ -52,13 +52,19 @@ create() -> %% on DER format to ssl:connect/listen.) ets:new(ssl_otp_cacertificate_db, [set, public]), %% Let connection processes call ref_count/3 directly - ets:new(ssl_otp_ca_file_ref, [set, public]), - ets:new(ssl_otp_pem_cache, [set, protected]), + {ets:new(ssl_otp_ca_file_ref, [set, public]), + ets:new(ssl_otp_ca_ref_file_mapping, [set, protected]) + }, + %% Lookups in named table owned by ssl_pem_cache process + ssl_otp_pem_cache, %% Default cache {ets:new(ssl_otp_crl_cache, [set, protected]), ets:new(ssl_otp_crl_issuer_mapping, [bag, protected])} ]. +create_pem_cache() -> + ets:new(ssl_otp_pem_cache, [named_table, set, protected]). + %%-------------------------------------------------------------------- -spec remove([db_handle()]) -> ok. %% @@ -70,6 +76,8 @@ remove(Dbs) -> true = ets:delete(Db1); (undefined) -> ok; + (ssl_otp_pem_cache) -> + ok; (Db) -> true = ets:delete(Db) end, Dbs). @@ -101,11 +109,6 @@ lookup_trusted_cert(_DbHandle, {extracted,Certs}, SerialNumber, Issuer) -> {ok, Cert} end. -lookup_cached_pem([_, _, PemChache | _], File) -> - lookup_cached_pem(PemChache, File); -lookup_cached_pem(PemChache, File) -> - lookup(File, PemChache). - %%-------------------------------------------------------------------- -spec add_trusted_certs(pid(), {erlang:timestamp(), string()} | {der, list()}, [db_handle()]) -> {ok, [db_handle()]}. @@ -122,17 +125,11 @@ add_trusted_certs(_Pid, {der, DerList}, [CertDb, _,_ | _]) -> add_certs_from_der(DerList, NewRef, CertDb), {ok, NewRef}; -add_trusted_certs(_Pid, File, [CertsDb, RefDb, PemChache | _] = Db) -> - case lookup_cached_pem(Db, File) of - [{_Content, Ref}] -> +add_trusted_certs(_Pid, File, [ _, {RefDb, FileMapDb} | _] = Db) -> + case lookup(File, FileMapDb) of + [Ref] -> ref_count(Ref, RefDb, 1), {ok, Ref}; - [Content] -> - Ref = make_ref(), - update_counter(Ref, 1, RefDb), - insert(File, {Content, Ref}, PemChache), - add_certs_from_pem(Content, Ref, CertsDb), - {ok, Ref}; undefined -> new_trusted_cert_entry(File, Db) end. @@ -151,25 +148,6 @@ extract_trusted_certs(File) -> {error, {badmatch, Error}} end. -%%-------------------------------------------------------------------- -%% -%% Description: Cache file as binary in DB -%%-------------------------------------------------------------------- --spec cache_pem_file(binary(), [db_handle()]) -> {ok, term()}. -cache_pem_file(File, [_CertsDb, _RefDb, PemChache | _]) -> - {ok, PemBin} = file:read_file(File), - Content = public_key:pem_decode(PemBin), - insert(File, Content, PemChache), - {ok, Content}. - - --spec cache_pem_file(reference(), binary(), [db_handle()]) -> {ok, term()}. -cache_pem_file(Ref, File, [_CertsDb, _RefDb, PemChache| _]) -> - {ok, PemBin} = file:read_file(File), - Content = public_key:pem_decode(PemBin), - insert(File, {Content, Ref}, PemChache), - {ok, Content}. - -spec decode_pem_file(binary()) -> {ok, term()}. decode_pem_file(File) -> case file:read_file(File) of @@ -246,6 +224,8 @@ select_cert_by_issuer(Cache, Issuer) -> %%-------------------------------------------------------------------- ref_count({extracted, _}, _Db, _N) -> not_cached; +ref_count(Key, {Db, _}, N) -> + ref_count(Key, Db, N); ref_count(Key, Db, N) -> ets:update_counter(Db,Key,N). @@ -278,9 +258,9 @@ insert(Key, Data, Db) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -update_counter(Key, Count, Db) -> - true = ets:insert(Db, {Key, Count}), - ok. +init_ref_db(Ref, File, {RefDb, FileMapDb}) -> + true = ets:insert(RefDb, {Ref, 1}), + true = ets:insert(FileMapDb, {File, Ref}). remove_certs(Ref, CertsDb) -> true = ets:match_delete(CertsDb, {{Ref, '_', '_'}, '_'}), @@ -326,10 +306,10 @@ decode_certs(Ref, Cert) -> undefined end. -new_trusted_cert_entry(File, [CertsDb, RefDb, _ | _] = Db) -> +new_trusted_cert_entry(File, [CertsDb, RefsDb, _ | _]) -> Ref = make_ref(), - update_counter(Ref, 1, RefDb), - {ok, Content} = cache_pem_file(Ref, File, Db), + init_ref_db(Ref, File, RefsDb), + {ok, Content} = ssl_pem_cache:insert(File), add_certs_from_pem(Content, Ref, CertsDb), {ok, Ref}. @@ -361,4 +341,3 @@ crl_issuer(DerCRL) -> CRL = public_key:der_decode('CertificateList', DerCRL), TBSCRL = CRL#'CertificateList'.tbsCertList, TBSCRL#'TBSCertList'.issuer. - diff --git a/lib/ssl/src/ssl_sup.erl b/lib/ssl/src/ssl_sup.erl index 8245801139..05a7aaaa82 100644 --- a/lib/ssl/src/ssl_sup.erl +++ b/lib/ssl/src/ssl_sup.erl @@ -25,7 +25,7 @@ -behaviour(supervisor). %% API --export([start_link/0, manager_opts/0]). +-export([start_link/0]). %% Supervisor callback -export([init/1]). @@ -44,90 +44,28 @@ start_link() -> %%%========================================================================= init([]) -> - SessionCertManager = session_and_cert_manager_child_spec(), - TLSConnetionManager = tls_connection_manager_child_spec(), - %% Handles emulated options so that they inherited by the accept - %% socket, even when setopts is performed on the listen socket - ListenOptionsTracker = listen_options_tracker_child_spec(), - - DTLSConnetionManager = dtls_connection_manager_child_spec(), - DTLSUdpListeners = dtls_udp_listeners_spec(), + {ok, {{rest_for_one, 10, 3600}, [ssl_admin_child_spec(), + ssl_connection_sup() + ]}}. - {ok, {{one_for_all, 10, 3600}, [SessionCertManager, TLSConnetionManager, - ListenOptionsTracker, - DTLSConnetionManager, DTLSUdpListeners - ]}}. - - -manager_opts() -> - CbOpts = case application:get_env(ssl, session_cb) of - {ok, Cb} when is_atom(Cb) -> - InitArgs = session_cb_init_args(), - [{session_cb, Cb}, {session_cb_init_args, InitArgs}]; - _ -> - [] - end, - case application:get_env(ssl, session_lifetime) of - {ok, Time} when is_integer(Time) -> - [{session_lifetime, Time}| CbOpts]; - _ -> - CbOpts - end. - %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- - -session_and_cert_manager_child_spec() -> - Opts = manager_opts(), - Name = ssl_manager, - StartFunc = {ssl_manager, start_link, [Opts]}, +ssl_admin_child_spec() -> + Name = ssl_admin_sup, + StartFunc = {ssl_admin_sup, start_link, []}, Restart = permanent, Shutdown = 4000, - Modules = [ssl_manager], - Type = worker, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -tls_connection_manager_child_spec() -> - Name = tls_connection, - StartFunc = {tls_connection_sup, start_link, []}, - Restart = permanent, - Shutdown = 4000, - Modules = [tls_connection_sup], + Modules = [ssl_admin_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -dtls_connection_manager_child_spec() -> - Name = dtls_connection, - StartFunc = {dtls_connection_sup, start_link, []}, +ssl_connection_sup() -> + Name = ssl_connection_sup, + StartFunc = {ssl_connection_sup, start_link, []}, Restart = permanent, Shutdown = 4000, - Modules = [dtls_connection_sup], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -listen_options_tracker_child_spec() -> - Name = tls_socket, - StartFunc = {ssl_listen_tracker_sup, start_link, []}, - Restart = permanent, - Shutdown = 4000, - Modules = [tls_socket], - Type = supervisor, - {Name, StartFunc, Restart, Shutdown, Type, Modules}. - -dtls_udp_listeners_spec() -> - Name = dtls_udp_listener, - StartFunc = {dtls_udp_sup, start_link, []}, - Restart = permanent, - Shutdown = 4000, - Modules = [], + Modules = [ssl_connection_sup], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -session_cb_init_args() -> - case application:get_env(ssl, session_cb_init_args) of - {ok, Args} when is_list(Args) -> - Args; - _ -> - [] - end. diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 32991d3079..77606911be 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -424,18 +424,26 @@ handle_common_event(internal, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, ssl_options = Options} = State0) -> try {Packets, Buf} = tls_handshake:get_tls_handshake(Version,Data,Buf0, Options), - State = + State1 = State0#state{protocol_buffers = Buffers#protocol_buffers{tls_handshake_buffer = Buf}}, - Events = tls_handshake_events(Packets), - case StateName of - connection -> - ssl_connection:hibernate_after(StateName, State, Events); - _ -> - {next_state, StateName, State#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} - end + case Packets of + [] -> + assert_buffer_sanity(Buf, Options), + {Record, State} = next_record(State1), + next_event(StateName, Record, State); + _ -> + Events = tls_handshake_events(Packets), + case StateName of + connection -> + ssl_connection:hibernate_after(StateName, State1, Events); + _ -> + {next_state, StateName, + State1#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} + end + end catch throw:#alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State0) + ssl_connection:handle_own_alert(Alert, Version, StateName, State0) end; %%% TLS record protocol level application data messages handle_common_event(internal, #ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, State) -> @@ -615,8 +623,6 @@ next_event(StateName, Record, State, Actions) -> {next_state, StateName, State, [{next_event, internal, Alert} | Actions]} end. -tls_handshake_events([]) -> - throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, malformed_handshake)); tls_handshake_events(Packets) -> lists:map(fun(Packet) -> {next_event, internal, {handshake, Packet}} @@ -735,3 +741,25 @@ unprocessed_events(Events) -> %% handshake events left to process before we should %% process more TLS-records received on the socket. erlang:length(Events)-1. + + +assert_buffer_sanity(<<?BYTE(_Type), ?UINT24(Length), Rest/binary>>, #ssl_options{max_handshake_size = Max}) when + Length =< Max -> + case size(Rest) of + N when N < Length -> + true; + N when N > Length -> + throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + too_big_handshake_data)); + _ -> + throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + malformed_handshake_data)) + end; +assert_buffer_sanity(Bin, _) -> + case size(Bin) of + N when N < 3 -> + true; + _ -> + throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + malformed_handshake_data)) + end. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 52c1af5b4c..f0a3c42e8d 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -136,7 +136,8 @@ options_tests() -> honor_server_cipher_order, honor_client_cipher_order, unordered_protocol_versions_server, - unordered_protocol_versions_client + unordered_protocol_versions_client, + max_handshake_size ]. options_tests_tls() -> @@ -960,9 +961,9 @@ clear_pem_cache(Config) when is_list(Config) -> {status, _, _, StatusInfo} = sys:get_status(whereis(ssl_manager)), [_, _,_, _, Prop] = StatusInfo, State = ssl_test_lib:state(Prop), - [_,FilRefDb |_] = element(6, State), + [_,{FilRefDb, _} |_] = element(6, State), {Server, Client} = basic_verify_test_no_close(Config), - CountReferencedFiles = fun({_,-1}, Acc) -> + CountReferencedFiles = fun({_, -1}, Acc) -> Acc; ({_, N}, Acc) -> N + Acc @@ -3860,6 +3861,29 @@ unordered_protocol_versions_client(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ServerMsg, Client, ClientMsg). %%-------------------------------------------------------------------- +max_handshake_size() -> + [{doc,"Test that we can set max_handshake_size to max value."}]. + +max_handshake_size(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{max_handshake_size, 8388607} |ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{max_handshake_size, 8388607} | ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok). + +%%-------------------------------------------------------------------- server_name_indication_option() -> [{doc,"Test API server_name_indication option to connect."}]. diff --git a/lib/ssl/test/ssl_pem_cache_SUITE.erl b/lib/ssl/test/ssl_pem_cache_SUITE.erl index f10d27fbc6..96b15d9b51 100644 --- a/lib/ssl/test/ssl_pem_cache_SUITE.erl +++ b/lib/ssl/test/ssl_pem_cache_SUITE.erl @@ -82,8 +82,8 @@ pem_cleanup() -> [{doc, "Test pem cache invalidate mechanism"}]. pem_cleanup(Config)when is_list(Config) -> process_flag(trap_exit, true), - ClientOpts = proplists:get_value(client_opts, Config), - ServerOpts = proplists:get_value(server_opts, Config), + ClientOpts = proplists:get_value(client_verification_opts, Config), + ServerOpts = proplists:get_value(server_verification_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 5656155c53..d2dd2848b5 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1558,13 +1558,17 @@ new_anno(Term) -> Abstr :: erl_parse_tree(). anno_to_term(Abstract) -> - map_anno(fun erl_anno:to_term/1, Abstract). + F = fun(Anno, Acc) -> {erl_anno:to_term(Anno), Acc} end, + {NewAbstract, []} = modify_anno1(Abstract, [], F), + NewAbstract. -spec anno_from_term(Term) -> erl_parse_tree() when Term :: term(). anno_from_term(Term) -> - map_anno(fun erl_anno:from_term/1, Term). + F = fun(T, Acc) -> {erl_anno:from_term(T), Acc} end, + {NewTerm, []} = modify_anno1(Term, [], F), + NewTerm. %% Forms. %% Recognize what sys_pre_expand does: diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 02b7cb10c2..47e7c4f03d 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -275,13 +275,13 @@ gen(_, _, Acc) -> lists:reverse(Acc). %% Check that the algorithms generate sound values. basic_stats_uniform_1(Config) when is_list(Config) -> - ct:timetrap({minutes,6}), %% valgrind needs a lot of time + ct:timetrap({minutes,15}), %% valgrind needs a lot of time [basic_uniform_1(?LOOP, rand:seed_s(Alg), 0.0, array:new([{default, 0}])) || Alg <- algs()], ok. basic_stats_uniform_2(Config) when is_list(Config) -> - ct:timetrap({minutes,6}), %% valgrind needs a lot of time + ct:timetrap({minutes,15}), %% valgrind needs a lot of time [basic_uniform_2(?LOOP, rand:seed_s(Alg), 0, array:new([{default, 0}])) || Alg <- algs()], ok. @@ -388,7 +388,7 @@ crypto_uniform_n(N, State0) -> %% Not a test but measures the time characteristics of the different algorithms measure(Suite) when is_atom(Suite) -> []; measure(_Config) -> - ct:timetrap({minutes,6}), %% valgrind needs a lot of time + ct:timetrap({minutes,15}), %% valgrind needs a lot of time Algos = [crypto64|algs()], io:format("RNG uniform integer performance~n",[]), _ = measure_1(random, fun(State) -> {int, random:uniform_s(10000, State)} end), diff --git a/lib/typer/src/typer.erl b/lib/typer/src/typer.erl index 5c82750a21..cd67af41ed 100644 --- a/lib/typer/src/typer.erl +++ b/lib/typer/src/typer.erl @@ -143,8 +143,9 @@ extract(#analysis{macros = Macros, MergedRecords = dialyzer_utils:merge_records(NewRecords, OldRecords), CodeServer2 = dialyzer_codeserver:set_temp_records(MergedRecords, CodeServer1), CodeServer3 = dialyzer_codeserver:finalize_exported_types(NewExpTypes, CodeServer2), - CodeServer4 = dialyzer_utils:process_record_remote_types(CodeServer3), - dialyzer_contracts:process_contract_remote_types(CodeServer4) + {CodeServer4, RecordDict} = + dialyzer_utils:process_record_remote_types(CodeServer3), + dialyzer_contracts:process_contract_remote_types(CodeServer4, RecordDict) catch throw:{error, ErrorMsg} -> compile_error(ErrorMsg) @@ -156,7 +157,7 @@ extract(#analysis{macros = Macros, fun(Module, TmpPlt) -> {ok, ModuleContracts} = dict:find(Module, Contracts), SpecList = [{MFA, Contract} - || {MFA, {_FileLine, Contract}} <- dict:to_list(ModuleContracts)], + || {MFA, {_FileLine, Contract}} <- maps:to_list(ModuleContracts)], dialyzer_plt:insert_contract_list(TmpPlt, SpecList) end, NewTrustPLT = lists:foldl(FoldFun, TrustPLT, Modules), @@ -172,8 +173,10 @@ get_type_info(#analysis{callgraph = CallGraph, StrippedCallGraph = remove_external(CallGraph, TrustPLT), %% io:format("--- Analyzing callgraph... "), try - NewPlt = dialyzer_succ_typings:analyze_callgraph(StrippedCallGraph, - TrustPLT, CodeServer), + NewMiniPlt = dialyzer_succ_typings:analyze_callgraph(StrippedCallGraph, + TrustPLT, + CodeServer), + NewPlt = dialyzer_plt:restore_full_plt(NewMiniPlt), Analysis#analysis{callgraph = StrippedCallGraph, trust_plt = NewPlt} catch error:What -> @@ -224,7 +227,7 @@ get_external(Exts, Plt) -> -type fa() :: {atom(), arity()}. -type func_info() :: {line(), atom(), arity()}. --record(info, {records = map__new() :: map_dict(), +-record(info, {records = maps:new() :: erl_types:type_table(), functions = [] :: [func_info()], types = map__new() :: map_dict(), edoc = false :: boolean()}). @@ -267,7 +270,7 @@ write_inc_files(Inc) -> Functions = [Key || {Key, _} <- Val], Val1 = [{{F,A},Type} || {{_Line,F,A},Type} <- Val], Info = #info{types = map__from_list(Val1), - records = map__new(), + records = maps:new(), %% Note we need to sort functions here! functions = lists:keysort(1, Functions)}, %% io:format("Types ~p\n", [Info#info.types]), @@ -849,8 +852,9 @@ collect_info(Analysis) -> TmpCServer1 = dialyzer_codeserver:set_temp_records(MergedRecords, TmpCServer), TmpCServer2 = dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), - TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2), - dialyzer_contracts:process_contract_remote_types(TmpCServer3) + {TmpCServer3, RecordDict} = + dialyzer_utils:process_record_remote_types(TmpCServer2), + dialyzer_contracts:process_contract_remote_types(TmpCServer3, RecordDict) catch throw:{error, ErrorMsg} -> fatal_error(ErrorMsg) diff --git a/lib/xmerl/src/xmerl_scan.erl b/lib/xmerl/src/xmerl_scan.erl index 5e0459ec21..9f6b27113e 100644 --- a/lib/xmerl/src/xmerl_scan.erl +++ b/lib/xmerl/src/xmerl_scan.erl @@ -2225,16 +2225,18 @@ processed_whole_element(S=#xmerl_scanner{hook_fun = _Hook, AllAttrs = case S#xmerl_scanner.default_attrs of true -> - [ #xmlAttribute{name = AttName, - parents = [{Name, Pos} | Parents], - language = Lang, - nsinfo = NSI, - namespace = Namespace, - value = AttValue, - normalized = true} || - {AttName, AttValue} <- get_default_attrs(S, Name), - AttValue =/= no_value, - not lists:keymember(AttName, #xmlAttribute.name, Attrs) ]; + DefaultAttrs = + [ #xmlAttribute{name = AttName, + parents = [{Name, Pos} | Parents], + language = Lang, + nsinfo = NSI, + namespace = Namespace, + value = AttValue, + normalized = true} || + {AttName, AttValue} <- get_default_attrs(S, Name), + AttValue =/= no_value, + not lists:keymember(AttName, #xmlAttribute.name, Attrs) ], + lists:append(Attrs, DefaultAttrs); false -> Attrs end, diff --git a/lib/xmerl/test/xmerl_SUITE.erl b/lib/xmerl/test/xmerl_SUITE.erl index e97b8c6a4b..cf7c0b7548 100644 --- a/lib/xmerl/test/xmerl_SUITE.erl +++ b/lib/xmerl/test/xmerl_SUITE.erl @@ -54,7 +54,8 @@ groups() -> cpd_expl_provided_DTD]}, {misc, [], [latin1_alias, syntax_bug1, syntax_bug2, syntax_bug3, - pe_ref1, copyright, testXSEIF, export_simple1, export]}, + pe_ref1, copyright, testXSEIF, export_simple1, export, + default_attrs_bug]}, {eventp_tests, [], [sax_parse_and_export]}, {ticket_tests, [], [ticket_5998, ticket_7211, ticket_7214, ticket_7430, @@ -223,6 +224,21 @@ syntax_bug3(Config) -> Err -> Err end. +default_attrs_bug(Config) -> + file:set_cwd(datadir(Config)), + Doc = "<!DOCTYPE doc [<!ATTLIST doc b CDATA \"default\">]>\n" + "<doc a=\"explicit\"/>", + {#xmlElement{attributes = [#xmlAttribute{name = a, value = "explicit"}, + #xmlAttribute{name = b, value = "default"}]}, + [] + } = xmerl_scan:string(Doc, [{default_attrs, true}]), + Doc2 = "<!DOCTYPE doc [<!ATTLIST doc b CDATA \"default\">]>\n" + "<doc b=\"also explicit\" a=\"explicit\"/>", + {#xmlElement{attributes = [#xmlAttribute{name = b, value = "also explicit"}, + #xmlAttribute{name = a, value = "explicit"}]}, + [] + } = xmerl_scan:string(Doc2, [{default_attrs, true}]). + pe_ref1(Config) -> file:set_cwd(datadir(Config)), {#xmlElement{},[]} = xmerl_scan:file(datadir_join(Config,[misc,"PE_ref1.xml"]),[{validation,true}]). diff --git a/otp_versions.table b/otp_versions.table index e21ca7fdf4..575a358010 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-19.2.1 : erts-8.2.1 # asn1-4.0.4 common_test-1.13 compiler-7.0.3 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.2 debugger-4.2.1 dialyzer-3.0.3 diameter-1.12.1 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.2 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.3 ic-4.4.2 inets-6.3.4 jinterface-1.7.1 kernel-5.1.1 megaco-3.18.1 mnesia-4.14.2 observer-2.3 odbc-2.12 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.3 reltool-0.7.2 runtime_tools-1.11 sasl-3.0.2 snmp-5.2.4 ssh-4.4 ssl-8.1 stdlib-3.2 syntax_tools-2.1.1 tools-2.9 typer-0.9.11 wx-1.8 xmerl-1.3.12 : OTP-19.2 : common_test-1.13 compiler-7.0.3 crypto-3.7.2 dialyzer-3.0.3 edoc-0.8.1 erl_docgen-0.6.1 erl_interface-3.9.2 erts-8.2 eunit-2.3.2 hipe-3.15.3 inets-6.3.4 kernel-5.1.1 mnesia-4.14.2 observer-2.3 odbc-2.12 parsetools-2.1.4 public_key-1.3 runtime_tools-1.11 sasl-3.0.2 ssh-4.4 ssl-8.1 stdlib-3.2 syntax_tools-2.1.1 tools-2.9 wx-1.8 # asn1-4.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 debugger-4.2.1 diameter-1.12.1 eldap-1.2.2 et-1.6 gs-1.6.2 ic-4.4.2 jinterface-1.7.1 megaco-3.18.1 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 percept-0.9 reltool-0.7.2 snmp-5.2.4 typer-0.9.11 xmerl-1.3.12 : OTP-19.1.6 : erts-8.1.1 # asn1-4.0.4 common_test-1.12.3 compiler-7.0.2 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.1 debugger-4.2.1 dialyzer-3.0.2 diameter-1.12.1 edoc-0.8 eldap-1.2.2 erl_docgen-0.6 erl_interface-3.9.1 et-1.6 eunit-2.3.1 gs-1.6.2 hipe-3.15.2 ic-4.4.2 inets-6.3.3 jinterface-1.7.1 kernel-5.1 megaco-3.18.1 mnesia-4.14.1 observer-2.2.2 odbc-2.11.3 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.3 percept-0.9 public_key-1.2 reltool-0.7.2 runtime_tools-1.10.1 sasl-3.0.1 snmp-5.2.4 ssh-4.3.6 ssl-8.0.3 stdlib-3.1 syntax_tools-2.1 tools-2.8.6 typer-0.9.11 wx-1.7.1 xmerl-1.3.12 : OTP-19.1.5 : ssh-4.3.6 # asn1-4.0.4 common_test-1.12.3 compiler-7.0.2 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.1 debugger-4.2.1 dialyzer-3.0.2 diameter-1.12.1 edoc-0.8 eldap-1.2.2 erl_docgen-0.6 erl_interface-3.9.1 erts-8.1 et-1.6 eunit-2.3.1 gs-1.6.2 hipe-3.15.2 ic-4.4.2 inets-6.3.3 jinterface-1.7.1 kernel-5.1 megaco-3.18.1 mnesia-4.14.1 observer-2.2.2 odbc-2.11.3 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.3 percept-0.9 public_key-1.2 reltool-0.7.2 runtime_tools-1.10.1 sasl-3.0.1 snmp-5.2.4 ssl-8.0.3 stdlib-3.1 syntax_tools-2.1 tools-2.8.6 typer-0.9.11 wx-1.7.1 xmerl-1.3.12 : |