diff options
90 files changed, 642 insertions, 603 deletions
diff --git a/.travis.yml b/.travis.yml index e05c644bd1..42151a16d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,14 +33,8 @@ script: after_success: - $ERL_TOP/bin/dialyzer --build_plt --apps asn1 compiler crypto dialyzer edoc erts et hipe inets kernel mnesia observer public_key runtime_tools snmp ssh ssl stdlib syntax_tools wx xmerl --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts kernel stdlib --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps asn1 crypto dialyzer --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps hipe parsetools public_key --statistics - - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps runtime_tools sasl tools --statistics - - $ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc --statistics - - $ERL_TOP/bin/dialyzer -n --apps inets mnesia observer --statistics - - $ERL_TOP/bin/dialyzer -n --apps ssh ssl syntax_tools --statistics - - $ERL_TOP/bin/dialyzer -n --apps tools wx xmerl --statistics + - $ERL_TOP/bin/dialyzer -n -Wunknown -Wunmatched_returns --apps compiler erts kernel stdlib asn1 crypto dialyzer hipe parsetools public_key runtime_tools sasl tools --statistics + - $ERL_TOP/bin/dialyzer -n --apps common_test debugger edoc inets mnesia observer ssh ssl syntax_tools tools wx xmerl --statistics - ./otp_build tests && make release_docs after_script: diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex 68dc2a5f61..b7bcadace0 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex 68dc2a5f61..b7bcadace0 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 588ab97968..8169505d78 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bsm.beam b/bootstrap/lib/compiler/ebin/beam_bsm.beam Binary files differindex cf5b3e2712..acba331476 100644 --- a/bootstrap/lib/compiler/ebin/beam_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_clean.beam b/bootstrap/lib/compiler/ebin/beam_clean.beam Binary files differindex ed0827953c..3d0d5112f8 100644 --- a/bootstrap/lib/compiler/ebin/beam_clean.beam +++ b/bootstrap/lib/compiler/ebin/beam_clean.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex 0f85080a31..3428b2872e 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_listing.beam b/bootstrap/lib/compiler/ebin/beam_listing.beam Binary files differindex 645440c85e..b80561b4dd 100644 --- a/bootstrap/lib/compiler/ebin/beam_listing.beam +++ b/bootstrap/lib/compiler/ebin/beam_listing.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex f8eb5e8a7c..9837ed458b 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index 237993f080..e91be8e4ad 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -19,7 +19,7 @@ {application, compiler, [{description, "ERTS CXC 138 10"}, - {vsn, "7.0.2"}, + {vsn, "7.0.3"}, {modules, [ beam_a, beam_asm, diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex a5545a1458..cd8fd5ac05 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam Binary files differindex 58dff1b796..eed59b6cfa 100644 --- a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam +++ b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam diff --git a/bootstrap/lib/compiler/ebin/v3_codegen.beam b/bootstrap/lib/compiler/ebin/v3_codegen.beam Binary files differindex 6cbe3165d1..1db120e00e 100644 --- a/bootstrap/lib/compiler/ebin/v3_codegen.beam +++ b/bootstrap/lib/compiler/ebin/v3_codegen.beam diff --git a/bootstrap/lib/kernel/ebin/heart.beam b/bootstrap/lib/kernel/ebin/heart.beam Binary files differindex 5a1cd86854..b5bb0806c8 100644 --- a/bootstrap/lib/kernel/ebin/heart.beam +++ b/bootstrap/lib/kernel/ebin/heart.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex ac0f44b7c8..078a0ffc9e 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.app b/bootstrap/lib/kernel/ebin/kernel.app index 60bb6e06ee..4af19d756e 100644 --- a/bootstrap/lib/kernel/ebin/kernel.app +++ b/bootstrap/lib/kernel/ebin/kernel.app @@ -22,7 +22,7 @@ {application, kernel, [ {description, "ERTS CXC 138 10"}, - {vsn, "5.1"}, + {vsn, "5.1.1"}, {modules, [application, application_controller, application_master, diff --git a/bootstrap/lib/stdlib/ebin/dict.beam b/bootstrap/lib/stdlib/ebin/dict.beam Binary files differindex 8beaf0022e..a4e8f674f0 100644 --- a/bootstrap/lib/stdlib/ebin/dict.beam +++ b/bootstrap/lib/stdlib/ebin/dict.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_eval.beam b/bootstrap/lib/stdlib/ebin/erl_eval.beam Binary files differindex e633572309..eff463adc0 100644 --- a/bootstrap/lib/stdlib/ebin/erl_eval.beam +++ b/bootstrap/lib/stdlib/ebin/erl_eval.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex c7262358ec..9f241899e0 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/escript.beam b/bootstrap/lib/stdlib/ebin/escript.beam Binary files differindex 096c28e969..2d04bab10c 100644 --- a/bootstrap/lib/stdlib/ebin/escript.beam +++ b/bootstrap/lib/stdlib/ebin/escript.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_trees.beam b/bootstrap/lib/stdlib/ebin/gb_trees.beam Binary files differindex 334e3649ae..a2627402ae 100644 --- a/bootstrap/lib/stdlib/ebin/gb_trees.beam +++ b/bootstrap/lib/stdlib/ebin/gb_trees.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_event.beam b/bootstrap/lib/stdlib/ebin/gen_event.beam Binary files differindex 023366da29..80db66b438 100644 --- a/bootstrap/lib/stdlib/ebin/gen_event.beam +++ b/bootstrap/lib/stdlib/ebin/gen_event.beam diff --git a/bootstrap/lib/stdlib/ebin/orddict.beam b/bootstrap/lib/stdlib/ebin/orddict.beam Binary files differindex 7ab6e61f6d..4624d2139a 100644 --- a/bootstrap/lib/stdlib/ebin/orddict.beam +++ b/bootstrap/lib/stdlib/ebin/orddict.beam diff --git a/bootstrap/lib/stdlib/ebin/stdlib.app b/bootstrap/lib/stdlib/ebin/stdlib.app index 916b7a0483..371ba299f0 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.app +++ b/bootstrap/lib/stdlib/ebin/stdlib.app @@ -20,7 +20,7 @@ %% {application, stdlib, [{description, "ERTS CXC 138 10"}, - {vsn, "3.1"}, + {vsn, "3.2"}, {modules, [array, base64, beam_lib, diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index ab00d47425..fe8e3b30e7 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -886,7 +886,7 @@ Rep(Fc) = <c>[Rep(C_1), ..., Rep(C_k)]</c>.</p> <list type="bulleted"> - <item>If C is a constraint <c>is_subtype(V, T)</c> or <c>V :: T</c>, + <item>If C is a constraint <c>V :: T</c>, where <c>V</c> is a type variable and <c>T</c> is a type, then Rep(C) = <c>{type,LINE,constraint,[{atom,LINE,is_subtype},[Rep(V),Rep(T)]]}</c>. diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 11777f0014..812538729d 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -32,6 +32,26 @@ <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 8.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix a quite rare bug causing VM crash during code loading + and the use of export funs (fun M:F/A) of not yet loaded + modules. Requires a very specfic timing of concurrent + scheduler threads. Has been seen on ARM but can probably + also occure on other architectures. Bug has existed since + OTP R16.</p> + <p> + Own Id: OTP-14144 Aux Id: seq13242 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 22576503ef..c8dde8caf8 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -807,7 +807,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } if (BIF_ARG_2 == am_true) { - int i; + int i, num_exps; /* * Make the code with the on_load function current. @@ -823,7 +823,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) /* * The on_load function succeded. Fix up export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; @@ -846,14 +847,15 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) hipe_redirect_to_module(modp); #endif } else if (BIF_ARG_2 == am_false) { - int i; + int i, num_exps; /* * The on_load function failed. Remove references to the * code that is about to be purged from the export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; @@ -1801,9 +1803,9 @@ delete_code(Module* modp) { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = make_atom(modp->module); - int i; + int i, num_exps = export_list_size(code_ix); - for (i = 0; i < export_list_size(code_ix); i++) { + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { if (ep->addressv[code_ix] == ep->beam) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 8f1faa6719..309465bcd3 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -800,14 +800,14 @@ erts_finish_loading(Binary* magic, Process* c_p, } else { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = stp->module; - int i; + int i, num_exps; /* * There is an -on_load() function. We will keep the current * code, but we must turn off any tracing. */ - - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep == NULL || ep->info.mfa.module != module) { continue; @@ -5724,12 +5724,13 @@ exported_from_module(Process* p, /* Process whose heap to use. */ ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { - int i; + int i, num_exps; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export* ep = export_list(i,code_ix); if (ep->info.mfa.module == mod) { diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 5b299d9acf..c4f75a887f 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -60,6 +60,10 @@ # define ERTS_GC_ASSERT(B) ((void) 1) #endif +#if defined(DEBUG) && 0 +# define HARDDEBUG 1 +#endif + /* * Returns number of elements in an array. */ @@ -111,7 +115,6 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static void deallocate_previous_young_generation(Process *c_p); static Eterm *full_sweep_heaps(Process *p, int hibernate, Eterm *n_heap, Eterm* n_htop, @@ -935,7 +938,7 @@ erts_garbage_collect_hibernate(Process* p) disallow_heap_frag_ref_in_heap(p, heap, htop); #endif - deallocate_previous_young_generation(p); + erts_deallocate_young_generation(p); p->heap = heap; p->high_water = htop; @@ -1628,7 +1631,7 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); #endif - deallocate_previous_young_generation(p); + erts_deallocate_young_generation(p); HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; @@ -1727,7 +1730,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); #endif - deallocate_previous_young_generation(p); + erts_deallocate_young_generation(p); HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; @@ -1889,8 +1892,8 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) return adjusted; } -static void -deallocate_previous_young_generation(Process *c_p) +void +erts_deallocate_young_generation(Process *c_p) { Eterm *orig_heap; diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 0dfdf52028..f4cbe732ce 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -29,10 +29,6 @@ #include "erl_map.h" -#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF -# define HARDDEBUG 1 -#endif - #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) @@ -160,5 +156,6 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); int erts_max_heap_size(Eterm, Uint *, Uint *); +void erts_deallocate_young_generation(Process *c_p); #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 8c9aa45a81..f80ebdf31b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -12695,7 +12695,6 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - Eterm *heap; ErtsPSD *psd; struct saved_calls *scb; process_breakpoint_time_t *pbt; @@ -12747,13 +12746,8 @@ delete_process(Process* p) hipe_delete_process(&p->hipe); #endif - heap = p->abandoned_heap ? p->abandoned_heap : p->heap; + erts_deallocate_young_generation(p); -#ifdef DEBUG - sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif - - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -12765,16 +12759,6 @@ delete_process(Process* p) (p->old_hend-p->old_heap)*sizeof(Eterm)); } - /* - * Free all pending message buffers. - */ - if (p->mbuf != NULL) { - free_message_buffer(p->mbuf); - } - - if (p->msg_frag) - erts_cleanup_messages(p->msg_frag); - erts_erase_dicts(p); /* free all pending messages */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index f397ab6b00..33ed6d7ec1 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -356,7 +356,7 @@ Export *export_list(int i, ErtsCodeIndex code_ix) int export_list_size(ErtsCodeIndex code_ix) { - return export_tables[code_ix].entries; + return erts_index_num_entries(&export_tables[code_ix]); } int export_table_sz(void) diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index c86a2122f6..cd834e2c12 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -91,9 +91,16 @@ index_put_entry(IndexTable* t, void* tmpl) t->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(t->type, sz); t->size += INDEX_PAGE_SIZE; } - t->entries++; p->index = ix; t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; + + /* + * Do a write barrier here to allow readers to do lock free iteration. + * erts_index_num_entries() does matching read barrier. + */ + ERTS_SMP_WRITE_MEMORY_BARRIER; + t->entries++; + return p; } diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index b2e3c0eab5..10f5d1eb39 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -65,6 +65,7 @@ void index_erase_latest_from(IndexTable*, Uint ix); ERTS_GLB_INLINE int index_put(IndexTable*, void*); ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint); +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -78,6 +79,19 @@ erts_index_lookup(IndexTable* t, Uint ix) { return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK]; } + +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t) +{ + int ret = t->entries; + /* + * Do a read barrier here to allow lock free iteration + * on tables where entries are never erased. + * index_put_entry() does matching write barrier. + */ + ERTS_SMP_READ_MEMORY_BARRIER; + return ret; +} + #endif #endif diff --git a/erts/vsn.mk b/erts/vsn.mk index af0be85062..028b114068 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 8.2 +VSN = 8.2.1 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/asn1/doc/src/Makefile b/lib/asn1/doc/src/Makefile index 559836116f..9a388e4e8a 100644 --- a/lib/asn1/doc/src/Makefile +++ b/lib/asn1/doc/src/Makefile @@ -37,8 +37,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN) # Target Specs # ---------------------------------------------------- XML_APPLICATION_FILES = ref_man.xml -XML_REF3_FILES = asn1ct.xml \ - asn1rt.xml +XML_REF3_FILES = asn1ct.xml GEN_XML = \ asn1_spec.xml diff --git a/lib/asn1/doc/src/asn1ct.xml b/lib/asn1/doc/src/asn1ct.xml index e5a7b1bcc4..ebe1ce44dc 100644 --- a/lib/asn1/doc/src/asn1ct.xml +++ b/lib/asn1/doc/src/asn1ct.xml @@ -321,45 +321,6 @@ File3.asn</pre> </func> <func> - <name>encode(Module, Type, Value)-> {ok, Bytes} | {error, Reason}</name> - <fsummary>Encodes an ASN.1 value.</fsummary> - <type> - <v>Module = Type = atom()</v> - <v>Value = term()</v> - <v>Bytes = binary()</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Encodes <c>Value</c> of <c>Type</c> defined in the <c>ASN.1</c> module - <c>Module</c>. To get as fast execution as possible, the - encode function performs only the rudimentary tests that input - <c>Value</c> is a correct instance of <c>Type</c>. So, for example, - the length of strings is - not always checked. Returns <c>{ok, Bytes}</c> if successful or - <c>{error, Reason}</c> if an error occurred. - </p> - <p>This function is deprecated. - Use <c>Module:encode(Type, Value)</c> instead.</p> - </desc> - </func> - - <func> - <name>decode(Module, Type, Bytes) -> {ok, Value} | {error, Reason}</name> - <fsummary>Decode from Bytes into an ASN.1 value.</fsummary> - <type> - <v>Module = Type = atom()</v> - <v>Value = Reason = term()</v> - <v>Bytes = binary()</v> - </type> - <desc> - <p>Decodes <c>Type</c> from <c>Module</c> from the binary - <c>Bytes</c>. Returns <c>{ok, Value}</c> if successful.</p> - <p>This function is deprecated. - Use <c>Module:decode(Type, Bytes)</c> instead.</p> - </desc> - </func> - - <func> <name>value(Module, Type) -> {ok, Value} | {error, Reason}</name> <fsummary>Creates an ASN.1 value for test purposes.</fsummary> <type> @@ -424,11 +385,11 @@ File3.asn</pre> <p>Schematically, the following occurs for each type in the module:</p> <code type="none"> {ok, Value} = asn1ct:value(Module, Type), -{ok, Bytes} = asn1ct:encode(Module, Type, Value), -{ok, Value} = asn1ct:decode(Module, Type, Bytes).</code> +{ok, Bytes} = Module:encode(Type, Value), +{ok, Value} = Module:decode(Type, Bytes).</code> <p>The <c>test</c> functions use the <c>*.asn1db</c> files for all included modules. If they are located in a different - directory than the current working directory, use the include + directory than the current working directory, use the <c>include</c> option to add paths. This is only needed when automatically generating values. For static values using <c>Value</c> no options are needed.</p> diff --git a/lib/asn1/doc/src/asn1rt.xml b/lib/asn1/doc/src/asn1rt.xml deleted file mode 100644 index 3f53ca0f56..0000000000 --- a/lib/asn1/doc/src/asn1rt.xml +++ /dev/null @@ -1,135 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>1997</year><year>2016</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>asn1rt</title> - <prepared>Kenneth Lundin</prepared> - <responsible>Kenneth Lundin</responsible> - <docno>1</docno> - <approved>Kenneth Lundin</approved> - <checked></checked> - <date>97-10-04</date> - <rev>A</rev> - <file>asn1.sgml</file> - </header> - <module>asn1rt</module> - <modulesummary>ASN.1 runtime support functions</modulesummary> - <description> - <warning> - <p> - All functions in this module are deprecated and will be - removed in a future release. - </p> - </warning> - </description> - - <funcs> - - <func> - <name>decode(Module,Type,Bytes) -> {ok,Value}|{error,Reason}</name> - <fsummary>Decodes from Bytes into an ASN.1 value.</fsummary> - <type> - <v>Module = Type = atom()</v> - <v>Value = Reason = term()</v> - <v>Bytes = binary</v> - </type> - <desc> - <p>Decodes <c>Type</c> from <c>Module</c> from the binary <c>Bytes</c>. - Returns <c>{ok,Value}</c> if successful.</p> - <p>Use <c>Module:decode(Type, Bytes)</c> instead of this function.</p> - </desc> - </func> - - <func> - <name>encode(Module,Type,Value)-> {ok,Bytes} | {error,Reason}</name> - <fsummary>Encodes an ASN.1 value.</fsummary> - <type> - <v>Module = Type = atom()</v> - <v>Value = term()</v> - <v>Bytes = binary</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Encodes <c>Value</c> of <c>Type</c> defined in the <c>ASN.1</c> - module <c>Module</c>. Returns a binary if successful. To get - as fast execution as possible, the encode function performs - only the rudimentary test that input <c>Value</c> is a correct - instance of <c>Type</c>. For example, the length of strings is - not always checked.</p> - <p>Use <c>Module:encode(Type, Value)</c> instead of this function.</p> - </desc> - </func> - - <func> - <name>info(Module) -> {ok,Info} | {error,Reason}</name> - <fsummary>Returns compiler information about the Module.</fsummary> - <type> - <v>Module = atom()</v> - <v>Info = list()</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Returns the version of the <c>ASN.1</c> compiler that was - used to compile the module. It also returns the compiler options - that were used.</p> - <p>Use <c>Module:info()</c> instead of this function.</p> - </desc> - </func> - - <func> - <name>utf8_binary_to_list(UTF8Binary) -> {ok,UnicodeList} | {error,Reason}</name> - <fsummary>Transforms an UTF8 encoded binary to a unicode list.</fsummary> - <type> - <v>UTF8Binary = binary()</v> - <v>UnicodeList = [integer()]</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Transforms a UTF8 encoded binary - to a list of integers, where each integer represents one - character as its unicode value. The function fails if the binary - is not a properly encoded UTF8 string.</p> - <p>Use <seealso marker="stdlib:unicode#characters_to_list-1">unicode:characters_to_list/1</seealso> instead of this function.</p> - </desc> - </func> - - <func> - <name>utf8_list_to_binary(UnicodeList) -> {ok,UTF8Binary} | {error,Reason}</name> - <fsummary>Transforms an unicode list to a UTF8 binary.</fsummary> - <type> - <v>UnicodeList = [integer()]</v> - <v>UTF8Binary = binary()</v> - <v>Reason = term()</v> - </type> - <desc> - <p>Transforms a list of integers, - where each integer represents one character as its unicode - value, to a UTF8 encoded binary.</p> - <p>Use <seealso marker="stdlib:unicode#characters_to_binary-1">unicode:characters_to_binary/1</seealso> instead of this function.</p> - </desc> - </func> - - </funcs> - -</erlref> - diff --git a/lib/asn1/src/Makefile b/lib/asn1/src/Makefile index 38cf2d496a..ba459f6cd3 100644 --- a/lib/asn1/src/Makefile +++ b/lib/asn1/src/Makefile @@ -68,7 +68,6 @@ CT_MODULES= \ $(EVAL_CT_MODULES) RT_MODULES= \ - asn1rt \ asn1rt_nif MODULES= $(CT_MODULES) $(RT_MODULES) diff --git a/lib/asn1/src/asn1.app.src b/lib/asn1/src/asn1.app.src index 1f8805ff5e..d2da727193 100644 --- a/lib/asn1/src/asn1.app.src +++ b/lib/asn1/src/asn1.app.src @@ -2,7 +2,6 @@ [{description, "The Erlang ASN1 compiler version %VSN%"}, {vsn, "%VSN%"}, {modules, [ - asn1rt, asn1rt_nif ]}, {registered, [ diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl index 8783b5418d..4e030861f5 100644 --- a/lib/asn1/src/asn1ct.erl +++ b/lib/asn1/src/asn1ct.erl @@ -20,17 +20,12 @@ %% %% -module(asn1ct). --deprecated([decode/3,encode/3]). --compile([{nowarn_deprecated_function,{asn1rt,decode,3}}, - {nowarn_deprecated_function,{asn1rt,encode,2}}, - {nowarn_deprecated_function,{asn1rt,encode,3}}]). %% Compile Time functions for ASN.1 (e.g ASN.1 compiler). %%-compile(export_all). %% Public exports -export([compile/1, compile/2]). --export([encode/2, encode/3, decode/3]). -export([test/1, test/2, test/3, value/2, value/3]). %% Application internal exports -export([compile_asn/3,compile_asn1/3,compile_py/3,compile/3, @@ -1271,21 +1266,6 @@ pretty2(Module,AbsFile) -> start(Includes) when is_list(Includes) -> asn1_db:dbstart(Includes). - -encode(Module,Term) -> - asn1rt:encode(Module,Term). - -encode(Module,Type,Term) when is_list(Module) -> - asn1rt:encode(list_to_atom(Module),Type,Term); -encode(Module,Type,Term) -> - asn1rt:encode(Module,Type,Term). - -decode(Module,Type,Bytes) when is_list(Module) -> - asn1rt:decode(list_to_atom(Module),Type,Bytes); -decode(Module,Type,Bytes) -> - asn1rt:decode(Module,Type,Bytes). - - test(Module) -> test_module(Module, []). test(Module, [] = Options) -> test_module(Module, Options); @@ -1330,10 +1310,10 @@ test_type(Module, Type) -> test_value(Module, Type, Value) -> in_process(fun() -> - case catch encode(Module, Type, Value) of + case catch Module:encode(Type, Value) of {ok, Bytes} -> NewBytes = prepare_bytes(Bytes), - case decode(Module, Type, NewBytes) of + case Module:decode(Type, NewBytes) of {ok, Value} -> {ok, {Module, Type, Value}}; {ok, Res} -> diff --git a/lib/asn1/src/asn1ct_value.erl b/lib/asn1/src/asn1ct_value.erl index 57cd3f8af6..b3d41dd9f3 100644 --- a/lib/asn1/src/asn1ct_value.erl +++ b/lib/asn1/src/asn1ct_value.erl @@ -19,7 +19,6 @@ %% %% -module(asn1ct_value). --compile([{nowarn_deprecated_function,{asn1rt,utf8_list_to_binary,1}}]). %% Generate Erlang values for ASN.1 types. %% The value is randomized within it's constraints @@ -292,8 +291,10 @@ from_type_prim(M, D) -> 'BMPString' -> adjust_list(size_random(C),c_string(C,"BMPString")); 'UTF8String' -> - {ok,Res}=asn1rt:utf8_list_to_binary(adjust_list(random(50),[$U,$T,$F,$8,$S,$t,$r,$i,$n,$g,16#ffff,16#fffffff,16#ffffff,16#fffff,16#fff])), - Res; + L = adjust_list(random(50), + [$U,$T,$F,$8,$S,$t,$r,$i,$n,$g, + 16#ffff,16#ffee,16#10ffff,16#ffff,16#fff]), + unicode:characters_to_binary(L); 'UniversalString' -> adjust_list(size_random(C),c_string(C,"UniversalString")); XX -> diff --git a/lib/asn1/src/asn1rt.erl b/lib/asn1/src/asn1rt.erl deleted file mode 100644 index 3e09ce2252..0000000000 --- a/lib/asn1/src/asn1rt.erl +++ /dev/null @@ -1,184 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%% --module(asn1rt). --deprecated(module). - -%% Runtime functions for ASN.1 (i.e encode, decode) - --export([encode/2,encode/3,decode/3,load_driver/0,unload_driver/0,info/1]). - --export([utf8_binary_to_list/1,utf8_list_to_binary/1]). - -encode(Module,{Type,Term}) -> - encode(Module,Type,Term). - -encode(Module,Type,Term) -> - case catch apply(Module,encode,[Type,Term]) of - {'EXIT',undef} -> - {error,{asn1,{undef,Module,Type}}}; - Result -> - Result - end. - -decode(Module,Type,Bytes) -> - case catch apply(Module,decode,[Type,Bytes]) of - {'EXIT',undef} -> - {error,{asn1,{undef,Module,Type}}}; - Result -> - Result - end. - -%% Remove in R16A -load_driver() -> - ok. - -unload_driver() -> - ok. - -info(Module) -> - case catch apply(Module,info,[]) of - {'EXIT',{undef,_Reason}} -> - {error,{asn1,{undef,Module,info}}}; - Result -> - {ok,Result} - end. - -%% utf8_binary_to_list/1 transforms a utf8 encoded binary to a list of -%% unicode elements, where each element is the unicode integer value -%% of a utf8 character. -%% Bin is a utf8 encoded value. The return value is either {ok,Val} or -%% {error,Reason}. Val is a list of integers, where each integer is a -%% unicode character value. -utf8_binary_to_list(Bin) when is_binary(Bin) -> - utf8_binary_to_list(Bin,[]). - -utf8_binary_to_list(<<>>,Acc) -> - {ok,lists:reverse(Acc)}; -utf8_binary_to_list(Bin,Acc) -> - Len = utf8_binary_len(Bin), - case catch split_binary(Bin,Len) of - {CharBin,RestBin} -> - case utf8_binary_char(CharBin) of - C when is_integer(C) -> - utf8_binary_to_list(RestBin,[C|Acc]); - Err -> Err - end; - Err -> {error,{asn1,{bad_encoded_utf8string,Err}}} - end. - -utf8_binary_len(<<0:1,_:7,_/binary>>) -> - 1; -utf8_binary_len(<<1:1,1:1,0:1,_:5,_/binary>>) -> - 2; -utf8_binary_len(<<1:1,1:1,1:1,0:1,_:4,_/binary>>) -> - 3; -utf8_binary_len(<<1:1,1:1,1:1,1:1,0:1,_:3,_/binary>>) -> - 4; -utf8_binary_len(<<1:1,1:1,1:1,1:1,1:1,0:1,_:2,_/binary>>) -> - 5; -utf8_binary_len(<<1:1,1:1,1:1,1:1,1:1,1:1,0:1,_:1,_/binary>>) -> - 6; -utf8_binary_len(Bin) -> - {error,{asn1,{bad_utf8_length,Bin}}}. - -utf8_binary_char(<<0:1,Int:7>>) -> - Int; -utf8_binary_char(<<_:2,0:1,Int1:5,1:1,0:1,Int2:6>>) -> - (Int1 bsl 6) bor Int2; -utf8_binary_char(<<_:3,0:1,Int1:4,1:1,0:1,Int2:6,1:1,0:1,Int3:6>>) -> - <<Res:16>> = <<Int1:4,Int2:6,Int3:6>>, - Res; -utf8_binary_char(<<_:4,0:1,Int1:3,Rest/binary>>) -> - <<1:1,0:1,Int2:6,1:1,0:1,Int3:6,1:1,0:1,Int4:6>> = Rest, - <<Res:24>> = <<0:3,Int1:3,Int2:6,Int3:6,Int4:6>>, - Res; -utf8_binary_char(<<_:5,0:1,Int1:2,Rest/binary>>) -> - <<1:1,0:1,Int2:6,1:1,0:1,Int3:6,1:1,0:1,Int4:6,1:1,0:1,Int5:6>> = Rest, - <<Res:32>> = <<0:6,Int1:2,Int2:6,Int3:6,Int4:6,Int5:6>>, - Res; -utf8_binary_char(<<_:6,0:1,I:1,Rest/binary>>) -> - <<1:1,0:1,Int2:6,1:1,0:1,Int3:6,1:1,0:1,Int4:6,1:1,0:1, - Int5:6,1:1,0:1,Int6:6>> = Rest, - <<Res:32>> = <<0:1,I:1,Int2:6,Int3:6,Int4:6,Int5:6,Int6:6>>, - Res; -utf8_binary_char(Err) -> - {error,{asn1,{bad_utf8_character_encoding,Err}}}. - - -%% macros used for utf8 encoding --define(bit1to6_into_utf8byte(I),16#80 bor (I band 16#3f)). --define(bit7to12_into_utf8byte(I),16#80 bor ((I band 16#fc0) bsr 6)). --define(bit13to18_into_utf8byte(I),16#80 bor ((I band 16#3f000) bsr 12)). --define(bit19to24_into_utf8byte(I),16#80 bor ((Int band 16#fc0000) bsr 18)). --define(bit25to30_into_utf8byte(I),16#80 bor ((Int band 16#3f000000) bsr 24)). - -%% utf8_list_to_binary/1 transforms a list of integers to a -%% binary. Each element in the input list has the unicode (integer) -%% value of an utf8 character. -%% The return value is either {ok,Bin} or {error,Reason}. The -%% resulting binary is utf8 encoded. -utf8_list_to_binary(List) -> - utf8_list_to_binary(List,[]). - -utf8_list_to_binary([],Acc) when is_list(Acc) -> - {ok,list_to_binary(lists:reverse(Acc))}; -utf8_list_to_binary([],Acc) -> - {error,{asn1,Acc}}; -utf8_list_to_binary([H|T],Acc) -> - case catch utf8_encode(H,Acc) of - NewAcc when is_list(NewAcc) -> - utf8_list_to_binary(T,NewAcc); - Err -> Err - end. - - -utf8_encode(Int,Acc) when Int < 128 -> - %% range 16#00000000 - 16#0000007f - %% utf8 encoding: 0xxxxxxx - [Int|Acc]; -utf8_encode(Int,Acc) when Int < 16#800 -> - %% range 16#00000080 - 16#000007ff - %% utf8 encoding: 110xxxxx 10xxxxxx - [?bit1to6_into_utf8byte(Int),16#c0 bor (Int bsr 6)|Acc]; -utf8_encode(Int,Acc) when Int < 16#10000 -> - %% range 16#00000800 - 16#0000ffff - %% utf8 encoding: 1110xxxx 10xxxxxx 10xxxxxx - [?bit1to6_into_utf8byte(Int),?bit7to12_into_utf8byte(Int), - 16#e0 bor ((Int band 16#f000) bsr 12)|Acc]; -utf8_encode(Int,Acc) when Int < 16#200000 -> - %% range 16#00010000 - 16#001fffff - %% utf8 encoding: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - [?bit1to6_into_utf8byte(Int),?bit7to12_into_utf8byte(Int), - ?bit13to18_into_utf8byte(Int), - 16#f0 bor ((Int band 16#1c0000) bsr 18)|Acc]; -utf8_encode(Int,Acc) when Int < 16#4000000 -> - %% range 16#00200000 - 16#03ffffff - %% utf8 encoding: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - [?bit1to6_into_utf8byte(Int),?bit7to12_into_utf8byte(Int), - ?bit13to18_into_utf8byte(Int),?bit19to24_into_utf8byte(Int), - 16#f8 bor ((Int band 16#3000000) bsr 24)|Acc]; -utf8_encode(Int,Acc) -> - %% range 16#04000000 - 16#7fffffff - %% utf8 encoding: 1111110x 10xxxxxx ...(total 6 bytes) 10xxxxxx - [?bit1to6_into_utf8byte(Int),?bit7to12_into_utf8byte(Int), - ?bit13to18_into_utf8byte(Int),?bit19to24_into_utf8byte(Int), - ?bit25to30_into_utf8byte(Int), - 16#fc bor ((Int band 16#40000000) bsr 30)|Acc]. diff --git a/lib/asn1/test/asn1_SUITE_data/extensionAdditionGroup.erl b/lib/asn1/test/asn1_SUITE_data/extensionAdditionGroup.erl index 6cf8ecf451..cd6c74b995 100644 --- a/lib/asn1/test/asn1_SUITE_data/extensionAdditionGroup.erl +++ b/lib/asn1/test/asn1_SUITE_data/extensionAdditionGroup.erl @@ -120,10 +120,10 @@ run3(Erule) -> asn1_NOVALUE,asn1_NOVALUE,asn1_NOVALUE}, asn1_NOVALUE,asn1_NOVALUE}}}}}}}, io:format("~p:~p~n",[Erule,Val]), - {ok,List}= asn1rt:encode('EUTRA-RRC-Definitions','DL-DCCH-Message',Val), + {ok,List}= 'EUTRA-RRC-Definitions':encode('DL-DCCH-Message',Val), Enc = iolist_to_binary(List), io:format("Result from encode:~n~p~n",[Enc]), - {ok,Val2} = asn1rt:decode('EUTRA-RRC-Definitions','DL-DCCH-Message',Enc), + {ok,Val2} = 'EUTRA-RRC-Definitions':decode('DL-DCCH-Message', Enc), io:format("Result from decode:~n~p~n",[Val2]), case Val2 of Val -> ok; diff --git a/lib/asn1/test/asn1_SUITE_data/testobj.erl b/lib/asn1/test/asn1_SUITE_data/testobj.erl index a0e00f8314..e547ea4572 100644 --- a/lib/asn1/test/asn1_SUITE_data/testobj.erl +++ b/lib/asn1/test/asn1_SUITE_data/testobj.erl @@ -1410,16 +1410,14 @@ int2bin(Int) -> %%%%%%%%%%%%%%%%% wrappers %%%%%%%%%%%%%%%%%%%%%%%% wrapper_encode(Module,Type,Value) -> - case asn1rt:encode(Module,Type,Value) of - {ok,X} when binary(X) -> + case Module:encode(Type, Value) of + {ok,X} when is_binary(X) -> {ok, binary_to_list(X)}; - {ok,X} -> - {ok, binary_to_list(list_to_binary(X))}; Error -> Error end. wrapper_decode(Module, Type, Bytes) when is_binary(Bytes) -> - asn1rt:decode(Module, Type, Bytes); + Module:decode(Type, Bytes); wrapper_decode(Module, Type, Bytes) when is_list(Bytes) -> - asn1rt:decode(Module, Type, list_to_binary(Bytes)). + Module:decode(Type, list_to_binary(Bytes)). diff --git a/lib/asn1/test/testPrimStrings.erl b/lib/asn1/test/testPrimStrings.erl index cb97655c15..b7f0323301 100644 --- a/lib/asn1/test/testPrimStrings.erl +++ b/lib/asn1/test/testPrimStrings.erl @@ -19,8 +19,6 @@ %% %% -module(testPrimStrings). --compile([{nowarn_deprecated_function,{asn1rt,utf8_list_to_binary,1}}, - {nowarn_deprecated_function,{asn1rt,utf8_binary_to_list,1}}]). -export([bit_string/2]). -export([octet_string/1]). @@ -756,19 +754,21 @@ utf8_string(_Rules) -> 16#800, 16#ffff, 16#10000, - 16#1fffff, - 16#200000, - 16#3ffffff, - 16#4000000, - 16#7fffffff], + 16#1ffff, + 16#20000, + 16#2ffff, + 16#e0000, + 16#effff, + 16#F0000, + 16#10ffff], [begin - {ok,UTF8} = asn1rt:utf8_list_to_binary([Char]), - {ok,[Char]} = asn1rt:utf8_binary_to_list(UTF8), + UTF8 = unicode:characters_to_binary([Char]), + [Char] = unicode:characters_to_list([UTF8]), roundtrip('UTF', UTF8) end || Char <- AllRanges], - {ok,UTF8} = asn1rt:utf8_list_to_binary(AllRanges), - {ok,AllRanges} = asn1rt:utf8_binary_to_list(UTF8), + UTF8 = unicode:characters_to_binary(AllRanges), + AllRanges = unicode:characters_to_list(UTF8), roundtrip('UTF', UTF8), ok. diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index c37f731d8c..cf60355a40 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -126,7 +126,7 @@ ERL_COMPILE_FLAGS += +native endif ERL_COMPILE_FLAGS += +inline +warn_unused_import \ -Werror \ - -I../../stdlib/include -I$(EGEN) -W + -I../../stdlib/include -I$(EGEN) -W +warn_missing_spec # ---------------------------------------------------- # Targets diff --git a/lib/compiler/src/beam_a.erl b/lib/compiler/src/beam_a.erl index 91e6d80da3..cdb32d5d55 100644 --- a/lib/compiler/src/beam_a.erl +++ b/lib/compiler/src/beam_a.erl @@ -25,6 +25,9 @@ -export([module/2]). +-spec module(beam_asm:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index 9c8ed2277f..a2f5dc674c 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -24,9 +24,42 @@ -export([module/4]). -export([encode/2]). +-export_type([fail/0,label/0,reg/0,src/0,module_code/0,function_name/0]). + -import(lists, [map/2,member/2,keymember/3,duplicate/2,splitwith/2]). -include("beam_opcodes.hrl"). +%% Common types for describing operands for BEAM instructions. +-type reg_num() :: 0..1023. +-type reg() :: {'x',reg_num()} | {'y',reg_num()}. +-type src() :: reg() | + {'literal',term()} | + {'atom',atom()} | + {'integer',integer()} | + 'nil' | + {'float',float()}. +-type label() :: pos_integer(). +-type fail() :: {'f',label() | 0}. + +%% asm_instruction() describes only the instructions that +%% are used in BEAM files (as opposed to internal instructions +%% used only during optimization). + +-type asm_instruction() :: atom() | tuple(). + +-type function_name() :: atom(). + +-type exports() :: [{function_name(),arity()}]. + +-type asm_function() :: + {'function',function_name(),arity(),label(),[asm_instruction()]}. + +-type module_code() :: + {module(),[_],[_],[asm_function()],pos_integer()}. + +-spec module(module_code(), exports(), [_], [compile:option()]) -> + {'ok',binary()}. + module(Code, Abst, SourceFile, Opts) -> {ok,assemble(Code, Abst, SourceFile, Opts)}. @@ -439,6 +472,8 @@ encode_alloc_list_1([{floats,Floats}|T], Dict, Acc0) -> encode_alloc_list_1([], Dict, Acc) -> {iolist_to_binary(Acc),Dict}. +-spec encode(non_neg_integer(), pos_integer()) -> iodata(). + encode(Tag, N) when N < 0 -> encode1(Tag, negative_to_bytes(N)); encode(Tag, N) when N < 16 -> diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index 6a35191f6e..6543e05e20 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -25,6 +25,9 @@ -export([module/2]). -import(lists, [reverse/1,reverse/2,foldl/3,member/2]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_bs.erl b/lib/compiler/src/beam_bs.erl index 2aed98d4e7..beb055b23d 100644 --- a/lib/compiler/src/beam_bs.erl +++ b/lib/compiler/src/beam_bs.erl @@ -25,6 +25,9 @@ -export([module/2]). -import(lists, [mapfoldl/3,reverse/1]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc0}, _Opt) -> {Fs,Lc} = mapfoldl(fun function/2, Lc0, Fs0), {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_bsm.erl b/lib/compiler/src/beam_bsm.erl index ae1b34ba49..9a4e7fb133 100644 --- a/lib/compiler/src/beam_bsm.erl +++ b/lib/compiler/src/beam_bsm.erl @@ -60,19 +60,26 @@ %%% data structures or passed to BIFs. %%% +-type label() :: beam_asm:label(). +-type func_info() :: {beam_asm:reg(),boolean()}. + -record(btb, - {f, %Gbtrees for all functions. - index, %{Label,Code} index (for liveness). - ok_br, %Labels that are OK. - must_not_save, %Must not save position when - % optimizing (reaches - % bs_context_to_binary). - must_save %Must save position when optimizing. + {f :: gb_trees:tree(label(), func_info()), + index :: beam_utils:code_index(), %{Label,Code} index (for liveness). + ok_br=gb_sets:empty() :: gb_sets:set(label()), %Labels that are OK. + must_not_save=false :: boolean(), %Must not save position when + % optimizing (reaches + % bs_context_to_binary). + must_save=false :: boolean() %Must save position when optimizing. }). + +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, Opts) -> - D = #btb{f=btb_index(Fs0)}, - Fs = [function(F, D) || F <- Fs0], + FIndex = btb_index(Fs0), + Fs = [function(F, FIndex) || F <- Fs0], Code = {Mod,Exp,Attr,Fs,Lc}, case proplists:get_bool(bin_opt_info, Opts) of true -> @@ -92,10 +99,10 @@ format_error({no_bin_opt,Reason}) -> %%% Local functions. %%% -function({function,Name,Arity,Entry,Is}, D0) -> +function({function,Name,Arity,Entry,Is}, FIndex) -> try Index = beam_utils:index_labels(Is), - D = D0#btb{index=Index}, + D = #btb{f=FIndex,index=Index}, {function,Name,Arity,Entry,btb_opt_1(Is, D, [])} catch Class:Error -> @@ -179,15 +186,14 @@ btb_gen_save(false, _, Acc) -> Acc. %% a bs_context_to_binary instruction. %% -btb_reaches_match(Is, RegList, D0) -> +btb_reaches_match(Is, RegList, D) -> try Regs = btb_regs_from_list(RegList), - D = D0#btb{ok_br=gb_sets:empty(),must_not_save=false,must_save=false}, #btb{must_not_save=MustNotSave,must_save=MustSave} = - btb_reaches_match_1(Is, Regs, D), - case MustNotSave and MustSave of + btb_reaches_match_1(Is, Regs, D), + case MustNotSave andalso MustSave of true -> btb_error(must_and_must_not_save); - _ -> {ok,MustSave} + false -> {ok,MustSave} end catch throw:{error,_}=Error -> Error diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index 10805a3c36..b736d39f9c 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -26,6 +26,9 @@ -export([clean_labels/1]). -import(lists, [map/2,foldl/3,reverse/1,filter/2]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,_}, Opts) -> Order = [Lbl || {function,_,_,Lbl,_} <- Fs0], All = foldl(fun({function,_,_,Lbl,_}=Func,D) -> dict:store(Lbl, Func, D) end, @@ -39,6 +42,10 @@ module({Mod,Exp,Attr,Fs0,_}, Opts) -> {ok,{Mod,Exp,Attr,Fs,Lc}}. %% Remove all bs_save2/2 instructions not referenced by a bs_restore2/2. + +-spec bs_clean_saves([beam_utils:instruction()]) -> + [beam_utils:instruction()]. + bs_clean_saves(Is) -> Needed = bs_restores(Is, []), bs_clean_saves_1(Is, gb_sets:from_list(Needed), []). @@ -98,13 +105,18 @@ add_to_work_list(F, {Fs,Used}=Sets) -> %%% want to see the expanded code in a .S file. %%% --record(st, {lmap, %Translation tables for labels. - entry, %Number of entry label. - lc %Label counter +-type label() :: beam_asm:label(). + +-record(st, {lmap :: [{label(),label()}], %Translation tables for labels. + entry :: beam_asm:label(), %Number of entry label. + lc :: non_neg_integer() %Label counter }). +-spec clean_labels([beam_utils:instruction()]) -> + {[beam_utils:instruction()],pos_integer()}. + clean_labels(Fs0) -> - St0 = #st{lmap=[],lc=1}, + St0 = #st{lmap=[],entry=1,lc=1}, {Fs1,#st{lmap=Lmap0,lc=Lc}} = function_renumber(Fs0, St0, []), Lmap = gb_trees:from_orddict(ordsets:from_list(Lmap0)), Fs = function_replace(Fs1, Lmap, []), diff --git a/lib/compiler/src/beam_dead.erl b/lib/compiler/src/beam_dead.erl index 9087586b58..d379fdc4eb 100644 --- a/lib/compiler/src/beam_dead.erl +++ b/lib/compiler/src/beam_dead.erl @@ -29,6 +29,10 @@ -import(lists, [mapfoldl/3,reverse/1]). + +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,_}, _Opts) -> {Fs1,Lc1} = beam_clean:clean_labels(Fs0), {Fs,Lc} = mapfoldl(fun function/2, Lc1, Fs1), diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl index 9565ab74c4..719d799fd7 100644 --- a/lib/compiler/src/beam_dict.erl +++ b/lib/compiler/src/beam_dict.erl @@ -28,7 +28,7 @@ string_table/1,lambda_table/1,literal_table/1, line_table/1]). --type label() :: non_neg_integer(). +-type label() :: beam_asm:label(). -type index() :: non_neg_integer(). @@ -38,13 +38,16 @@ -type line_tab() :: #{{Fname :: index(), Line :: term()} => index()}. -type literal_tab() :: dict:dict(Literal :: term(), index()). +-type lambda_info() :: {label(),{index(),label(),non_neg_integer()}}. +-type lambda_tab() :: {non_neg_integer(),[lambda_info()]}. + -record(asm, {atoms = #{} :: atom_tab(), exports = [] :: [{label(), arity(), label()}], locals = [] :: [{label(), arity(), label()}], imports = gb_trees:empty() :: import_tab(), strings = <<>> :: binary(), %String pool - lambdas = {0,[]}, %[{...}] + lambdas = {0,[]} :: lambda_tab(), literals = dict:new() :: literal_tab(), fnames = #{} :: fname_tab(), lines = #{} :: line_tab(), @@ -148,10 +151,7 @@ string(Str, Dict) when is_list(Str) -> lambda(Lbl, NumFree, #asm{lambdas={OldIndex,Lambdas0}}=Dict) -> %% Set Index the same as OldIndex. Index = OldIndex, - %% Initialize OldUniq to 0. It will be set to an unique value - %% based on the MD5 checksum of the BEAM code for the module. - OldUniq = 0, - Lambdas = [{Lbl,{OldIndex,Lbl,Index,NumFree,OldUniq}}|Lambdas0], + Lambdas = [{Lbl,{Index,Lbl,NumFree}}|Lambdas0], {OldIndex,Dict#asm{lambdas={OldIndex+1,Lambdas}}}. %% Returns the index for a literal (adding it to the literal table if necessary). @@ -185,6 +185,9 @@ line([{location,Name,Line}], #asm{lines=Lines,num_lines=N}=Dict0) -> {Index, Dict1#asm{lines=Lines#{Key=>Index},num_lines=N+1}} end. +-spec fname(nonempty_string(), bdict()) -> + {non_neg_integer(), bdict()}. + fname(Name, #asm{fnames=Fnames}=Dict) -> case Fnames of #{Name := Index} -> {Index,Dict}; @@ -239,8 +242,11 @@ lambda_table(#asm{locals=Loc0,lambdas={NumLambdas,Lambdas0}}) -> Lambdas1 = sofs:relation(Lambdas0), Loc = sofs:relation([{Lbl,{F,A}} || {F,A,Lbl} <- Loc0]), Lambdas2 = sofs:relative_product1(Lambdas1, Loc), + %% Initialize OldUniq to 0. It will be set to an unique value + %% based on the MD5 checksum of the BEAM code for the module. + OldUniq = 0, Lambdas = [<<F:32,A:32,Lbl:32,Index:32,NumFree:32,OldUniq:32>> || - {{_,Lbl,Index,NumFree,OldUniq},{F,A}} <- sofs:to_external(Lambdas2)], + {{Index,Lbl,NumFree},{F,A}} <- sofs:to_external(Lambdas2)], {NumLambdas,Lambdas}. %% Returns the literal table. diff --git a/lib/compiler/src/beam_except.erl b/lib/compiler/src/beam_except.erl index 4a181c1923..9801c68ee2 100644 --- a/lib/compiler/src/beam_except.erl +++ b/lib/compiler/src/beam_except.erl @@ -33,6 +33,9 @@ -import(lists, [reverse/1]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. @@ -49,9 +52,9 @@ function({function,Name,Arity,CLabel,Is0}) -> end. -record(st, - {lbl, %func_info label - loc, %location for func_info - arity %arity for function + {lbl :: beam_asm:label(), %func_info label + loc :: [_], %location for func_info + arity :: arity() %arity for function }). function_1(Is0) -> diff --git a/lib/compiler/src/beam_flatten.erl b/lib/compiler/src/beam_flatten.erl index c9ff07b496..a4d45a4ca6 100644 --- a/lib/compiler/src/beam_flatten.erl +++ b/lib/compiler/src/beam_flatten.erl @@ -25,6 +25,9 @@ -import(lists, [reverse/1,reverse/2]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs,Lc}, _Opt) -> {ok,{Mod,Exp,Attr,[function(F) || F <- Fs],Lc}}. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index e096270d8c..4365451356 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -130,6 +130,11 @@ -import(lists, [reverse/1,reverse/2,foldl/3]). +-type instruction() :: beam_utils:instruction(). + +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. @@ -269,9 +274,9 @@ extract_seq_1(_, _) -> no. -record(st, { - entry, %Entry label (must not be moved). - mlbl, %Moved labels. - labels :: cerl_sets:set() %Set of referenced labels. + entry :: beam_asm:label(), %Entry label (must not be moved). + mlbl :: #{beam_asm:label() := [beam_asm:label()]}, %Moved labels. + labels :: cerl_sets:set() %Set of referenced labels. }). opt(Is0, CLabel) -> @@ -453,6 +458,8 @@ is_label_used(L, St) -> %% is_unreachable_after(Instruction) -> boolean() %% Test whether the code after Instruction is unreachable. +-spec is_unreachable_after(instruction()) -> boolean(). + is_unreachable_after({func_info,_M,_F,_A}) -> true; is_unreachable_after(return) -> true; is_unreachable_after({jump,_Lbl}) -> true; @@ -465,6 +472,8 @@ is_unreachable_after(I) -> is_exit_instruction(I). %% Test whether the instruction Instruction always %% causes an exit/failure. +-spec is_exit_instruction(instruction()) -> boolean(). + is_exit_instruction({call_ext,_,{extfunc,M,F,A}}) -> erl_bifs:is_exit_bif(M, F, A); is_exit_instruction(if_end) -> true; @@ -477,6 +486,8 @@ is_exit_instruction(_) -> false. %% Remove all unused labels. Also remove unreachable %% instructions following labels that are removed. +-spec remove_unused_labels([instruction()]) -> [instruction()]. + remove_unused_labels(Is) -> Used0 = initial_labels(Is), Used = foldl(fun ulbl/2, Used0, Is), diff --git a/lib/compiler/src/beam_listing.erl b/lib/compiler/src/beam_listing.erl index d82ed8639d..94b47cf568 100644 --- a/lib/compiler/src/beam_listing.erl +++ b/lib/compiler/src/beam_listing.erl @@ -21,14 +21,24 @@ -export([module/2]). +-include("core_parse.hrl"). +-include("v3_kernel.hrl"). -include("v3_life.hrl"). -import(lists, [foreach/2]). -module(File, Core) when element(1, Core) == c_module -> +-type code() :: cerl:c_module() + | beam_utils:module_code() + | #k_mdef{} + | {module(),_,_,_} %v3_life + | [_]. %form-based format + +-spec module(file:io_device(), code()) -> 'ok'. + +module(File, #c_module{}=Core) -> %% This is a core module. io:put_chars(File, core_pp:format(Core)); -module(File, Kern) when element(1, Kern) == k_mdef -> +module(File, #k_mdef{}=Kern) -> %% This is a kernel module. io:put_chars(File, v3_kernel_pp:format(Kern)); %%io:put_chars(File, io_lib:format("~p~n", [Kern])); diff --git a/lib/compiler/src/beam_peep.erl b/lib/compiler/src/beam_peep.erl index c8bef31824..6df5c02334 100644 --- a/lib/compiler/src/beam_peep.erl +++ b/lib/compiler/src/beam_peep.erl @@ -24,6 +24,9 @@ -import(lists, [reverse/1,member/2]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,_}, _Opts) -> %% First coalesce adjacent labels. {Fs1,Lc} = beam_clean:clean_labels(Fs0), diff --git a/lib/compiler/src/beam_receive.erl b/lib/compiler/src/beam_receive.erl index 89cafe27ce..1403e1e05e 100644 --- a/lib/compiler/src/beam_receive.erl +++ b/lib/compiler/src/beam_receive.erl @@ -65,6 +65,9 @@ %%% as the SomeUniqInteger. %%% +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> Fs = [function(F) || F <- Fs0], Code = {Mod,Exp,Attr,Fs,Lc}, diff --git a/lib/compiler/src/beam_reorder.erl b/lib/compiler/src/beam_reorder.erl index 6a7c033ec6..910b7f6b0a 100644 --- a/lib/compiler/src/beam_reorder.erl +++ b/lib/compiler/src/beam_reorder.erl @@ -23,6 +23,9 @@ -export([module/2]). -import(lists, [member/2,reverse/1]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_split.erl b/lib/compiler/src/beam_split.erl index feeab0af50..d041f18806 100644 --- a/lib/compiler/src/beam_split.erl +++ b/lib/compiler/src/beam_split.erl @@ -23,6 +23,9 @@ -import(lists, [reverse/1]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> Fs = [split_blocks(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_trim.erl b/lib/compiler/src/beam_trim.erl index d40669083e..4da0985085 100644 --- a/lib/compiler/src/beam_trim.erl +++ b/lib/compiler/src/beam_trim.erl @@ -24,10 +24,13 @@ -import(lists, [reverse/1,reverse/2,splitwith/2,sort/1]). -record(st, - {safe, %Safe labels. - lbl %Code at each label. + {safe :: gb_sets:set(beam_asm:label()), %Safe labels. + lbl :: beam_utils:code_index() %Code at each label. }). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 9866dcd070..050c599d6b 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -26,6 +26,9 @@ -import(lists, [filter/2,foldl/3,keyfind/3,member/2, reverse/1,reverse/2,sort/1]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index ffeff9ea81..cc6e54ca16 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -28,11 +28,31 @@ live_opt/1,delete_live_annos/1,combine_heap_needs/2, split_even/1]). +-export_type([code_index/0,module_code/0,instruction/0]). + -import(lists, [member/2,sort/1,reverse/1,splitwith/2]). +%% instruction() describes all instructions that are used during optimzation +%% (from beam_a to beam_z). +-type instruction() :: atom() | tuple(). + +-type code_index() :: gb_trees:tree(beam_asm:label(), [instruction()]). + +-type int_function() :: {'function',beam_asm:function_name(),arity(), + beam_asm:label(),[instruction()]}. + +-type module_code() :: + {module(),[_],[_],[int_function()],pos_integer()}. + +%% Internal types. +-type fail() :: beam_asm:fail() | 'fail'. +-type test() :: {'test',atom(),fail(),[beam_asm:src()]} | + {'test',atom(),fail(),integer(),list(),beam_asm:reg()}. +-type result_cache() :: gb_trees:tree(beam_asm:label(), 'killed' | 'used'). + -record(live, - {lbl, %Label to code index. - res}). %Result cache for each label. + {lbl :: code_index(), %Label to code index. + res :: result_cache()}). %Result cache for each label. %% is_killed_block(Register, [Instruction]) -> true|false @@ -44,6 +64,8 @@ %% i.e. it is OK to enter the instruction sequence with Register %% containing garbage. +-spec is_killed_block(beam_asm:reg(), [instruction()]) -> boolean(). + is_killed_block({x,X}, [{set,_,_,{alloc,Live,_}}|_]) -> X >= Live; is_killed_block(R, [{set,Ds,Ss,_Op}|Is]) -> @@ -65,6 +87,8 @@ is_killed_block(_, []) -> false. %% The state (constructed by index_instructions/1) is used to allow us %% to determine the kill state across branches. +-spec is_killed(beam_asm:reg(), [instruction()], code_index()) -> boolean(). + is_killed(R, Is, D) -> St = #live{lbl=D,res=gb_trees:empty()}, case check_liveness(R, Is, St) of @@ -75,6 +99,8 @@ is_killed(R, Is, D) -> %% is_killed_at(Reg, Lbl, State) -> true|false %% Determine whether Reg is killed at label Lbl. +-spec is_killed_at(beam_asm:reg(), beam_asm:label(), code_index()) -> boolean(). + is_killed_at(R, Lbl, D) when is_integer(Lbl) -> St0 = #live{lbl=D,res=gb_trees:empty()}, case check_liveness_at(R, Lbl, St0) of @@ -89,6 +115,8 @@ is_killed_at(R, Lbl, D) when is_integer(Lbl) -> %% The state is used to allow us to determine the usage state %% across branches. +-spec is_not_used(beam_asm:reg(), [instruction()], code_index()) -> boolean(). + is_not_used(R, Is, D) -> St = #live{lbl=D,res=gb_trees:empty()}, case check_liveness(R, Is, St) of @@ -100,18 +128,25 @@ is_not_used(R, Is, D) -> %% Index the instruction sequence so that we can quickly %% look up the instruction following a specific label. +-spec index_labels([instruction()]) -> code_index(). + index_labels(Is) -> index_labels_1(Is, []). %% empty_label_index() -> State %% Create an empty label index. +-spec empty_label_index() -> code_index(). + empty_label_index() -> gb_trees:empty(). %% index_label(Label, [Instruction], State) -> State %% Add an index for a label. +-spec index_label(beam_asm:label(), [instruction()], code_index()) -> + code_index(). + index_label(Lbl, Is0, Acc) -> Is = drop_labels(Is0), gb_trees:enter(Lbl, Is, Acc). @@ -120,12 +155,16 @@ index_label(Lbl, Is0, Acc) -> %% code_at(Label, State) -> [I]. %% Retrieve the code at the given label. +-spec code_at(beam_asm:label(), code_index()) -> [instruction()]. + code_at(L, Ll) -> gb_trees:get(L, Ll). %% bif_to_test(Bif, [Op], Fail) -> {test,Test,Fail,[Op]} %% Convert a BIF to a test. Fail if not possible. +-spec bif_to_test(atom(), list(), fail()) -> test(). + bif_to_test(is_atom, [_]=Ops, Fail) -> {test,is_atom,Fail,Ops}; bif_to_test(is_boolean, [_]=Ops, Fail) -> {test,is_boolean,Fail,Ops}; bif_to_test(is_binary, [_]=Ops, Fail) -> {test,is_binary,Fail,Ops}; @@ -158,6 +197,9 @@ bif_to_test(is_record, [_,_,_]=Ops, Fail) -> {test,is_record,Fail,Ops}. %% Return 'true' if the test instruction does not modify any %% registers and/or bit syntax matching state. %% + +-spec is_pure_test(test()) -> boolean(). + is_pure_test({test,is_eq,_,[_,_]}) -> true; is_pure_test({test,is_ne,_,[_,_]}) -> true; is_pure_test({test,is_eq_exact,_,[_,_]}) -> true; @@ -180,7 +222,9 @@ is_pure_test({test,Op,_,Ops}) -> %% whose destination is a register that will not be used. %% Also insert {'%live',Live,Regs} annotations at the beginning %% and end of each block. -%% + +-spec live_opt([instruction()]) -> [instruction()]. + live_opt(Is0) -> {[{label,Fail}|_]=Bef,[Fi|Is]} = splitwith(fun({func_info,_,_,_}) -> false; @@ -193,7 +237,9 @@ live_opt(Is0) -> %% delete_live_annos([Instruction]) -> [Instruction]. %% Delete all live annotations. -%% + +-spec delete_live_annos([instruction()]) -> [instruction()]. + delete_live_annos([{block,Bl0}|Is]) -> case delete_live_annos(Bl0) of [] -> delete_live_annos(Is); @@ -208,6 +254,8 @@ delete_live_annos([]) -> []. %% combine_heap_needs(HeapNeed1, HeapNeed2) -> HeapNeed %% Combine the heap need for two allocation instructions. +-spec combine_heap_needs(term(), term()) -> term(). + combine_heap_needs({alloc,Alloc1}, {alloc,Alloc2}) -> {alloc,combine_alloc_lists(Alloc1, Alloc2)}; combine_heap_needs({alloc,Alloc}, Words) when is_integer(Words) -> @@ -220,6 +268,8 @@ combine_heap_needs(H1, H2) when is_integer(H1), is_integer(H2) -> %% split_even/1 %% [1,2,3,4,5,6] -> {[1,3,5],[2,4,6]} +-spec split_even(list()) -> {list(),list()}. + split_even(Rs) -> split_even(Rs, [], []). diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 5659077c5d..bf33ae0aeb 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -32,6 +32,10 @@ -import(lists, [reverse/1,foldl/3,foreach/2,dropwhile/2]). %% To be called by the compiler. + +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_utils:module_code()}. + module({Mod,Exp,Attr,Fs,Lc}=Code, _Opts) when is_atom(Mod), is_list(Exp), is_list(Attr), is_integer(Lc) -> case validate(Mod, Fs) of diff --git a/lib/compiler/src/beam_z.erl b/lib/compiler/src/beam_z.erl index 6c7f8543c2..787e33c142 100644 --- a/lib/compiler/src/beam_z.erl +++ b/lib/compiler/src/beam_z.erl @@ -26,6 +26,9 @@ -import(lists, [dropwhile/2]). +-spec module(beam_utils:module_code(), [compile:option()]) -> + {'ok',beam_asm:module_code()}. + module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> Fs = [function(F) || F <- Fs0], {ok,{Mod,Exp,Attr,Fs,Lc}}. diff --git a/lib/compiler/src/cerl.erl b/lib/compiler/src/cerl.erl index b1be6ffc6d..6b936a7687 100644 --- a/lib/compiler/src/cerl.erl +++ b/lib/compiler/src/cerl.erl @@ -1584,6 +1584,8 @@ ann_make_list(_, [], Node) -> %% @doc Returns <code>true</code> if <code>Node</code> is an abstract %% map constructor, otherwise <code>false</code>. +-type map_op() :: #c_literal{val::'assoc'} | #c_literal{val::'exact'}. + -spec is_c_map(cerl()) -> boolean(). is_c_map(#c_map{}) -> @@ -1679,8 +1681,16 @@ update_c_map(#c_map{is_pat=true}=Old, M, Es) -> update_c_map(#c_map{is_pat=false}=Old, M, Es) -> ann_c_map(get_ann(Old), M, Es). +-spec map_pair_key(c_map_pair()) -> cerl(). + map_pair_key(#c_map_pair{key=K}) -> K. + +-spec map_pair_val(c_map_pair()) -> cerl(). + map_pair_val(#c_map_pair{val=V}) -> V. + +-spec map_pair_op(c_map_pair()) -> map_op(). + map_pair_op(#c_map_pair{op=Op}) -> Op. -spec c_map_pair(cerl(), cerl()) -> c_map_pair(). @@ -1699,6 +1709,8 @@ c_map_pair_exact(Key,Val) -> ann_c_map_pair(As,Op,K,V) -> #c_map_pair{op=Op, key = K, val=V, anno = As}. +-spec update_c_map_pair(c_map_pair(), map_op(), cerl(), cerl()) -> c_map_pair(). + update_c_map_pair(Old,Op,K,V) -> #c_map_pair{op=Op, key=K, val=V, anno = get_ann(Old)}. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index a28cd193c7..069add7890 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -228,6 +228,8 @@ expand_opt(O, Os) -> [O|Os]. %% format_error(ErrorDescriptor) -> string() +-spec format_error(term()) -> iolist(). + format_error(no_native_support) -> "this system is not configured for native-code compilation."; format_error(no_crypto) -> @@ -288,20 +290,22 @@ format_error_reason({Reason, Stack}) when is_list(Stack) -> format_error_reason(Reason) -> io_lib:format("~tp", [Reason]). +-type err_warn_info() :: tuple(). + %% The compile state record. -record(compile, {filename="" :: file:filename(), dir="" :: file:filename(), base="" :: file:filename(), ifile="" :: file:filename(), ofile="" :: file:filename(), - module=[], - core_code=[], - abstract_code=[], %Abstract code for debugger. - options=[] :: [option()], %Options for compilation + module=[] :: module() | [], + core_code=[] :: cerl:c_module() | [], + abstract_code=[] :: binary() | [], %Abstract code for debugger. + options=[] :: [option()], %Options for compilation mod_options=[] :: [option()], %Options for module_info encoding=none :: none | epp:source_encoding(), - errors=[], - warnings=[]}). + errors=[] :: [err_warn_info()], + warnings=[] :: [err_warn_info()]}). internal({forms,Forms}, Opts0) -> {_,Ps} = passes(forms, Opts0), @@ -1600,6 +1604,9 @@ list_errors(_F, []) -> ok. %% tmpfile(ObjFile) -> TmpFile %% Work out the correct input and output file names. +-spec iofile(atom() | file:filename_all()) -> + {file:name_all(),file:name_all()}. + iofile(File) when is_atom(File) -> iofile(atom_to_list(File)); iofile(File) -> @@ -1734,6 +1741,8 @@ help(_) -> %% compile(AbsFileName, Outfilename, Options) %% Compile entry point for erl_compile. +-spec compile(file:filename(), _, #options{}) -> 'ok' | 'error'. + compile(File0, _OutFile, Options) -> pre_load(), File = shorten_filename(File0), @@ -1742,6 +1751,8 @@ compile(File0, _OutFile, Options) -> Other -> Other end. +-spec compile_beam(file:filename(), _, #options{}) -> 'ok' | 'error'. + compile_beam(File0, _OutFile, Opts) -> File = shorten_filename(File0), case file(File, [from_beam|make_erl_options(Opts)]) of @@ -1749,6 +1760,8 @@ compile_beam(File0, _OutFile, Opts) -> Other -> Other end. +-spec compile_asm(file:filename(), _, #options{}) -> 'ok' | 'error'. + compile_asm(File0, _OutFile, Opts) -> File = shorten_filename(File0), case file(File, [from_asm|make_erl_options(Opts)]) of @@ -1756,6 +1769,8 @@ compile_asm(File0, _OutFile, Opts) -> Other -> Other end. +-spec compile_core(file:filename(), _, #options{}) -> 'ok' | 'error'. + compile_core(File0, _OutFile, Opts) -> File = shorten_filename(File0), case file(File, [from_core|make_erl_options(Opts)]) of diff --git a/lib/compiler/src/core_scan.erl b/lib/compiler/src/core_scan.erl index 11b52f6c5f..15bfc78c8b 100644 --- a/lib/compiler/src/core_scan.erl +++ b/lib/compiler/src/core_scan.erl @@ -49,13 +49,37 @@ -import(lists, [reverse/1]). +-type location() :: integer(). +-type category() :: atom(). +-type symbol() :: atom() | float() | integer() | string(). +-type token() :: {category(), Anno :: location(), symbol()} + | {category(), Anno :: location()}. +-type tokens() :: [token()]. +-type error_description() :: term(). +-type error_info() :: {erl_anno:location(), module(), error_description()}. + %% string([Char]) -> %% string([Char], StartPos) -> %% {ok, [Tok], EndPos} | %% {error, {Pos,core_scan,What}, EndPos} +-spec string(String) -> Return when + String :: string(), + Return :: {'ok', Tokens :: tokens(), EndLocation} + | {'error', ErrorInfo :: error_info(), ErrorLocation}, + EndLocation :: location(), + ErrorLocation :: location(). + string(Cs) -> string(Cs, 1). +-spec string(String, StartLocation) -> Return when + String :: string(), + Return :: {'ok', Tokens :: tokens(), EndLocation} + | {'error', ErrorInfo :: error_info(), ErrorLocation}, + StartLocation :: location(), + EndLocation :: location(), + ErrorLocation :: location(). + string(Cs, Sp) -> %% Add an 'eof' to always get correct handling. case string_pre_scan(Cs, [], Sp) of diff --git a/lib/compiler/src/sys_pre_attributes.erl b/lib/compiler/src/sys_pre_attributes.erl index bc93c85989..67adae5acf 100644 --- a/lib/compiler/src/sys_pre_attributes.erl +++ b/lib/compiler/src/sys_pre_attributes.erl @@ -25,10 +25,10 @@ -define(OPTION_TAG, attributes). --record(state, {forms, - pre_ops = [], - post_ops = [], - options}). +-record(state, {forms :: [form()], + pre_ops = [] :: [op()], + post_ops = [] :: [op()], + options :: [option()]}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Inserts, deletes and replaces Erlang compiler attributes. @@ -59,9 +59,23 @@ %% due to that the pre_transform pass did not find the attribute plus %% all insert operations. +-type attribute() :: atom(). +-type value() :: term(). +-type form() :: {function, integer(), atom(), arity(), _} + | {attribute, integer(), attribute(), _}. +-type option() :: compile:option() + | {'attribute', 'insert', attribute(), value()} + | {'attribute', 'replace', attribute(), value()} + | {'attribute', 'delete', attribute()}. +-type op() :: {'insert', attribute(), value()} + | {'replace', attribute(), value()} + | {'delete', attribute()}. + +-spec parse_transform([form()], [option()]) -> [form()]. + parse_transform(Forms, Options) -> S = #state{forms = Forms, options = Options}, - S2 = init_transform(S), + S2 = init_transform(Options, S), report_verbose("Pre options: ~p~n", [S2#state.pre_ops], S2), report_verbose("Post options: ~p~n", [S2#state.post_ops], S2), S3 = pre_transform(S2), @@ -71,13 +85,6 @@ parse_transform(Forms, Options) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Computes the lists of pre_ops and post_ops that are %% used in the real transformation. -init_transform(S) -> - case S#state.options of - Options when is_list(Options) -> - init_transform(Options, S); - Option -> - init_transform([Option], S) - end. init_transform([{attribute, insert, Name, Val} | Tail], S) -> Op = {insert, Name, Val}, @@ -92,12 +99,9 @@ init_transform([{attribute, delete, Name} | Tail], S) -> Op = {delete, Name}, PreOps = [Op | S#state.pre_ops], init_transform(Tail, S#state{pre_ops = PreOps}); -init_transform([], S) -> - S; init_transform([_ | T], S) -> init_transform(T, S); -init_transform(BadOpt, S) -> - report_error("Illegal option (ignored): ~p~n", [BadOpt], S), +init_transform([], S) -> S. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -176,18 +180,9 @@ attrs([], _, _) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Report functions. %% -%% Errors messages are controlled with the 'report_errors' compiler option %% Warning messages are controlled with the 'report_warnings' compiler option %% Verbose messages are controlled with the 'verbose' compiler option -report_error(Format, Args, S) -> - case is_error(S) of - true -> - io:format("~p: * ERROR * " ++ Format, [?MODULE | Args]); - false -> - ok - end. - report_warning(Format, Args, S) -> case is_warning(S) of true -> @@ -204,9 +199,6 @@ report_verbose(Format, Args, S) -> ok end. -is_error(S) -> - lists:member(report_errors, S#state.options) or is_verbose(S). - is_warning(S) -> lists:member(report_warnings, S#state.options) or is_verbose(S). diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index 3627cdb7cd..47c1567f10 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -69,6 +69,10 @@ stk=[], %Stack table res=[]}). %Reserved regs: [{reserved,I,V}] +-type life_module() :: {module(),_,_,[_]}. + +-spec module(life_module(), [compile:option()]) -> {'ok',beam_asm:module_code()}. + module({Mod,Exp,Attr,Forms}, _Options) -> {Fs,St} = functions(Forms, {atom,Mod}), {ok,{Mod,Exp,Attr,Fs,St#cg.lcount}}. diff --git a/lib/compiler/src/v3_kernel_pp.erl b/lib/compiler/src/v3_kernel_pp.erl index d5f6ee19c9..187e69a22c 100644 --- a/lib/compiler/src/v3_kernel_pp.erl +++ b/lib/compiler/src/v3_kernel_pp.erl @@ -47,7 +47,7 @@ canno(Cthing) -> element(2, Cthing). --spec format(cerl:cerl()) -> iolist(). +-spec format(#k_mdef{}) -> iolist(). format(Node) -> format(Node, #ctxt{}). diff --git a/lib/compiler/src/v3_life.erl b/lib/compiler/src/v3_life.erl index 0f2aeda87f..be3ade47ff 100644 --- a/lib/compiler/src/v3_life.erl +++ b/lib/compiler/src/v3_life.erl @@ -52,10 +52,15 @@ -include("v3_kernel.hrl"). -include("v3_life.hrl"). +-type fa() :: {atom(),arity()}. + %% These are not defined in v3_kernel.hrl. get_kanno(Kthing) -> element(2, Kthing). %%set_kanno(Kthing, Anno) -> setelement(2, Kthing, Anno). +-spec module(#k_mdef{}, [compile:option()]) -> + {'ok',{module(),[fa()],[_],[_]}}. + module(#k_mdef{name=M,exports=Es,attributes=As,body=Fs0}, _Opts) -> Fs1 = functions(Fs0, []), {ok,{M,Es,As,Fs1}}. @@ -416,6 +421,10 @@ add_var(V, F, L, Vdb) -> vdb_new(Vs) -> sort([{V,0,0} || {var,V} <- Vs]). +-type var() :: atom(). + +-spec vdb_find(var(), [vdb_entry()]) -> 'error' | vdb_entry(). + vdb_find(V, Vdb) -> case lists:keyfind(V, 1, Vdb) of false -> error; diff --git a/lib/compiler/src/v3_life.hrl b/lib/compiler/src/v3_life.hrl index 9d03a86ccd..5c76312067 100644 --- a/lib/compiler/src/v3_life.hrl +++ b/lib/compiler/src/v3_life.hrl @@ -20,8 +20,10 @@ %% This record contains variable life-time annotation for a %% kernel expression. Added by v3_life, used by v3_codegen. +-type vdb_entry() :: {atom(),non_neg_integer(),non_neg_integer()}. + -record(l, {ke, %Kernel expression - i=0, %Op number - vdb=[], %Variable database - a}). %Core annotation + i=0 :: non_neg_integer(), %Op number + vdb=[] :: [vdb_entry()], %Variable database + a=[] :: [term()]}). %Core annotation diff --git a/lib/dialyzer/test/small_SUITE_data/src/anno.erl b/lib/dialyzer/test/small_SUITE_data/src/anno.erl new file mode 100644 index 0000000000..70f1d42141 --- /dev/null +++ b/lib/dialyzer/test/small_SUITE_data/src/anno.erl @@ -0,0 +1,18 @@ +-module(anno). + +%% OTP-14131 + +-export([t1/0, t2/0, t3/0]). + +t1() -> + A = erl_parse:anno_from_term({attribute, 1, module, my_test}), + compile:forms([A], []). + +t2() -> + A = erl_parse:new_anno({attribute, 1, module, my_test}), + compile:forms([A], []). + +t3() -> + A = erl_parse:new_anno({attribute, 1, module, my_test}), + T = erl_parse:anno_to_term(A), + {attribute, 1, module, my_test} = T. diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index edc7e0d8b2..916b41742e 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -424,6 +424,14 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid </taglist> </item> + + <tag><c>max_handshake_size</c></tag> + <item> + <p>Integer (24 bits unsigned). Used to limit the size of + valid TLS handshake packets to avoid DoS attacks. + Defaults to 256*1024.</p> + </item> + </taglist> </item> diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index c72ee44a95..0b7229b67e 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -765,7 +765,8 @@ handle_options(Opts0, Role) -> client, Role), crl_check = handle_option(crl_check, Opts, false), crl_cache = handle_option(crl_cache, Opts, {ssl_crl_cache, {internal, []}}), - v2_hello_compatible = handle_option(v2_hello_compatible, Opts, false) + v2_hello_compatible = handle_option(v2_hello_compatible, Opts, false), + max_handshake_size = handle_option(max_handshake_size, Opts, ?DEFAULT_MAX_HANDSHAKE_SIZE) }, CbInfo = proplists:get_value(cb_info, Opts, default_cb_info(Protocol)), @@ -780,7 +781,8 @@ handle_options(Opts0, Role) -> alpn_preferred_protocols, next_protocols_advertised, client_preferred_next_protocols, log_alert, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, - fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, v2_hello_compatible], + fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, v2_hello_compatible, + max_handshake_size], SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) @@ -1028,6 +1030,8 @@ validate_option(beast_mitigation, Value) when Value == one_n_minus_one orelse Value; validate_option(v2_hello_compatible, Value) when is_boolean(Value) -> Value; +validate_option(max_handshake_size, Value) when is_integer(Value) andalso Value =< ?MAX_UNIT24 -> + Value; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index fde92035a2..324b7dbde3 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -80,6 +80,9 @@ -define(CLIENT_KEY_EXCHANGE, 16). -define(FINISHED, 20). +-define(MAX_UNIT24, 8388607). +-define(DEFAULT_MAX_HANDSHAKE_SIZE, (256*1024)). + -record(random, { gmt_unix_time, % uint32 random_bytes % opaque random_bytes[28] diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 98b89bb811..c34af9f82c 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -142,7 +142,8 @@ signature_algs, eccs, honor_ecc_order :: boolean(), - v2_hello_compatible :: boolean() + v2_hello_compatible :: boolean(), + max_handshake_size :: integer() }). -record(socket_options, diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 32991d3079..77606911be 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -424,18 +424,26 @@ handle_common_event(internal, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, ssl_options = Options} = State0) -> try {Packets, Buf} = tls_handshake:get_tls_handshake(Version,Data,Buf0, Options), - State = + State1 = State0#state{protocol_buffers = Buffers#protocol_buffers{tls_handshake_buffer = Buf}}, - Events = tls_handshake_events(Packets), - case StateName of - connection -> - ssl_connection:hibernate_after(StateName, State, Events); - _ -> - {next_state, StateName, State#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} - end + case Packets of + [] -> + assert_buffer_sanity(Buf, Options), + {Record, State} = next_record(State1), + next_event(StateName, Record, State); + _ -> + Events = tls_handshake_events(Packets), + case StateName of + connection -> + ssl_connection:hibernate_after(StateName, State1, Events); + _ -> + {next_state, StateName, + State1#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} + end + end catch throw:#alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State0) + ssl_connection:handle_own_alert(Alert, Version, StateName, State0) end; %%% TLS record protocol level application data messages handle_common_event(internal, #ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, State) -> @@ -615,8 +623,6 @@ next_event(StateName, Record, State, Actions) -> {next_state, StateName, State, [{next_event, internal, Alert} | Actions]} end. -tls_handshake_events([]) -> - throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, malformed_handshake)); tls_handshake_events(Packets) -> lists:map(fun(Packet) -> {next_event, internal, {handshake, Packet}} @@ -735,3 +741,25 @@ unprocessed_events(Events) -> %% handshake events left to process before we should %% process more TLS-records received on the socket. erlang:length(Events)-1. + + +assert_buffer_sanity(<<?BYTE(_Type), ?UINT24(Length), Rest/binary>>, #ssl_options{max_handshake_size = Max}) when + Length =< Max -> + case size(Rest) of + N when N < Length -> + true; + N when N > Length -> + throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + too_big_handshake_data)); + _ -> + throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + malformed_handshake_data)) + end; +assert_buffer_sanity(Bin, _) -> + case size(Bin) of + N when N < 3 -> + true; + _ -> + throw(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE, + malformed_handshake_data)) + end. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 52c1af5b4c..de5895d7ba 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -136,7 +136,8 @@ options_tests() -> honor_server_cipher_order, honor_client_cipher_order, unordered_protocol_versions_server, - unordered_protocol_versions_client + unordered_protocol_versions_client, + max_handshake_size ]. options_tests_tls() -> @@ -3860,6 +3861,29 @@ unordered_protocol_versions_client(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ServerMsg, Client, ClientMsg). %%-------------------------------------------------------------------- +max_handshake_size() -> + [{doc,"Test that we can set max_handshake_size to max value."}]. + +max_handshake_size(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{max_handshake_size, 8388607} |ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{max_handshake_size, 8388607} | ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok). + +%%-------------------------------------------------------------------- server_name_indication_option() -> [{doc,"Test API server_name_indication option to connect."}]. diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index fb5d05ec8e..922455a6f2 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ list tail list_comprehension lc_expr lc_exprs binary_comprehension tuple -%struct record_expr record_tuple record_field record_fields map_expr map_tuple map_field map_field_assoc map_field_exact map_fields map_key if_expr if_clause if_clauses case_expr cr_clause cr_clauses receive_expr @@ -108,9 +107,8 @@ type_sig -> fun_type 'when' type_guards : {type, ?anno('$1'), bounded_fun, type_guards -> type_guard : ['$1']. type_guards -> type_guard ',' type_guards : ['$1'|'$3']. -type_guard -> atom '(' top_types ')' : {type, ?anno('$1'), constraint, - ['$1', '$3']}. -type_guard -> var '::' top_type : build_def('$1', '$3'). +type_guard -> atom '(' top_types ')' : build_compat_constraint('$1', '$3'). +type_guard -> var '::' top_type : build_constraint('$1', '$3'). top_types -> top_type : ['$1']. top_types -> top_type ',' top_types : ['$1'|'$3']. @@ -269,7 +267,6 @@ expr_max -> binary : '$1'. expr_max -> list_comprehension : '$1'. expr_max -> binary_comprehension : '$1'. expr_max -> tuple : '$1'. -%%expr_max -> struct : '$1'. expr_max -> '(' expr ')' : '$2'. expr_max -> 'begin' exprs 'end' : {block,?anno('$1'),'$2'}. expr_max -> if_expr : '$1'. @@ -328,10 +325,6 @@ lc_expr -> binary '<=' expr : {b_generate,?anno('$2'),'$1','$3'}. tuple -> '{' '}' : {tuple,?anno('$1'),[]}. tuple -> '{' exprs '}' : {tuple,?anno('$1'),'$2'}. - -%%struct -> atom tuple : -%% {struct,?anno('$1'),element(3, '$1'),element(3, '$2')}. - map_expr -> '#' map_tuple : {map, ?anno('$1'),'$2'}. map_expr -> expr_max '#' map_tuple : @@ -1057,13 +1050,13 @@ build_typed_attribute({atom,Aa,Attr},_) -> end. build_type_spec({Kind,Aa}, {SpecFun, TypeSpecs}) - when (Kind =:= spec) or (Kind =:= callback) -> + when Kind =:= spec ; Kind =:= callback -> NewSpecFun = case SpecFun of {atom, _, Fun} -> {Fun, find_arity_from_specs(TypeSpecs)}; - {{atom,_, Mod}, {atom,_, Fun}} -> - {Mod,Fun,find_arity_from_specs(TypeSpecs)} + {{atom, _, Mod}, {atom, _, Fun}} -> + {Mod, Fun, find_arity_from_specs(TypeSpecs)} end, {attribute,Aa,Kind,{NewSpecFun, TypeSpecs}}. @@ -1077,11 +1070,24 @@ find_arity_from_specs([Spec|_]) -> {type, _, 'fun', [{type, _, product, Args},_]} = Fun, length(Args). -build_def({var, A, '_'}, _Types) -> +%% The 'is_subtype(V, T)' syntax is not supported as of Erlang/OTP +%% 19.0, but is kept for backward compatibility. +build_compat_constraint({atom, _, is_subtype}, [{var, _, _}=LHS, Type]) -> + build_constraint(LHS, Type); +build_compat_constraint({atom, _, is_subtype}, [LHS, _Type]) -> + ret_err(?anno(LHS), "bad type variable"); +build_compat_constraint({atom, A, Atom}, _Types) -> + ret_err(A, io_lib:format("unsupported constraint ~w", [Atom])). + +build_constraint({atom, _, is_subtype}, [{var, _, _}=LHS, Type]) -> + build_constraint(LHS, Type); +build_constraint({atom, A, Atom}, _Foo) -> + ret_err(A, io_lib:format("unsupported constraint ~w", [Atom])); +build_constraint({var, A, '_'}, _Types) -> ret_err(A, "bad type variable"); -build_def(LHS, Types) -> +build_constraint(LHS, Type) -> IsSubType = {atom, ?anno(LHS), is_subtype}, - {type, ?anno(LHS), constraint, [IsSubType, [LHS, Types]]}. + {type, ?anno(LHS), constraint, [IsSubType, [LHS, Type]]}. lift_unions(T1, {type, _Aa, union, List}) -> {type, ?anno(T1), union, [T1|List]}; @@ -1574,13 +1580,17 @@ new_anno(Term) -> Abstr :: erl_parse_tree(). anno_to_term(Abstract) -> - map_anno(fun erl_anno:to_term/1, Abstract). + F = fun(Anno, Acc) -> {erl_anno:to_term(Anno), Acc} end, + {NewAbstract, []} = modify_anno1(Abstract, [], F), + NewAbstract. -spec anno_from_term(Term) -> erl_parse_tree() when Term :: term(). anno_from_term(Term) -> - map_anno(fun erl_anno:from_term/1, Term). + F = fun(T, Acc) -> {erl_anno:from_term(T), Acc} end, + {NewTerm, []} = modify_anno1(Term, [], F), + NewTerm. %% Forms. modify_anno1({function,F,A}, Ac, _Mf) -> diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 4161ced9ab..f4257fb571 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -408,7 +408,7 @@ obsolete_1(docb_xml_check, _, _) -> %% Added in R15B obsolete_1(asn1rt, F, _) when F == load_driver; F == unload_driver -> - {deprecated,"deprecated (will be removed in OTP 18); has no effect as drivers are no longer used"}; + {removed,"removed (will be removed in OTP 18); has no effect as drivers are no longer used"}; obsolete_1(ssl, pid, 1) -> {removed,"was removed in R16; is no longer needed"}; obsolete_1(inviso, _, _) -> @@ -463,21 +463,23 @@ obsolete_1(wxCursor, new, 4) -> %% Added in OTP 17. obsolete_1(asn1ct, decode,3) -> - {deprecated,"deprecated; use Mod:decode/2 instead"}; + {removed,"removed; use Mod:decode/2 instead"}; +obsolete_1(asn1ct, encode, 2) -> + {removed,"removed; use Mod:encode/2 instead"}; obsolete_1(asn1ct, encode, 3) -> - {deprecated,"deprecated; use Mod:encode/2 instead"}; + {removed,"removed; use Mod:encode/2 instead"}; obsolete_1(asn1rt, decode,3) -> - {deprecated,"deprecated; use Mod:decode/2 instead"}; + {removed,"removed; use Mod:decode/2 instead"}; obsolete_1(asn1rt, encode, 2) -> - {deprecated,"deprecated; use Mod:encode/2 instead"}; + {removed,"removed; use Mod:encode/2 instead"}; obsolete_1(asn1rt, encode, 3) -> - {deprecated,"deprecated; use Mod:encode/2 instead"}; + {removed,"removed; use Mod:encode/2 instead"}; obsolete_1(asn1rt, info, 1) -> - {deprecated,"deprecated; use Mod:info/0 instead"}; + {removed,"removed; use Mod:info/0 instead"}; obsolete_1(asn1rt, utf8_binary_to_list, 1) -> - {deprecated,{unicode,characters_to_list,1}}; + {removed,{unicode,characters_to_list,1},"OTP 20"}; obsolete_1(asn1rt, utf8_list_to_binary, 1) -> - {deprecated,{unicode,characters_to_binary,1}}; + {removed,{unicode,characters_to_binary,1},"OTP 20"}; %% Added in OTP 18. obsolete_1(core_lib, get_anno, 1) -> diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index c86e17f70c..c90f855b3b 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -64,7 +64,7 @@ predef/1, maps/1,maps_type/1,maps_parallel_match/1, otp_11851/1,otp_11879/1,otp_13230/1, - record_errors/1]). + record_errors/1, otp_xxxxx/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -84,7 +84,7 @@ all() -> too_many_arguments, basic_errors, bin_syntax_errors, predef, maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, - record_errors]. + record_errors, otp_xxxxx]. groups() -> [{unused_vars_warn, [], @@ -3869,6 +3869,55 @@ record_errors(Config) when is_list(Config) -> {3,erl_lint,{redefine_field,r,a}}],[]}}], run(Config, Ts). +otp_xxxxx(Config) -> + Ts = [{constraint1, + <<"-export([t/1]). + -spec t(X) -> X when is_subtype(integer()). + t(a) -> foo:bar(). + ">>, + [], + {errors, + [{2,erl_parse,"unsupported constraint " ++ ["is_subtype"]}], + []}}, + {constraint2, + <<"-export([t/1]). + -spec t(X) -> X when bad_atom(X, integer()). + t(a) -> foo:bar(). + ">>, + [], + {errors, + [{2,erl_parse,"unsupported constraint " ++ ["bad_atom"]}], + []}}, + {constraint3, + <<"-export([t/1]). + -spec t(X) -> X when is_subtype(bad_variable, integer()). + t(a) -> foo:bar(). + ">>, + [], + {errors,[{2,erl_parse,"bad type variable"}],[]}}, + {constraint4, + <<"-export([t/1]). + -spec t(X) -> X when is_subtype(atom(), integer()). + t(a) -> foo:bar(). + ">>, + [], + {errors,[{2,erl_parse,"bad type variable"}],[]}}, + {constraint5, + <<"-export([t/1]). + -spec t(X) -> X when is_subtype(X, integer()). + t(a) -> foo:bar(). + ">>, + [], + []}, + {constraint6, + <<"-export([t/1]). + -spec t(X) -> X when X :: integer(). + t(a) -> foo:bar(). + ">>, + [], + []}], + run(Config, Ts). + run(Config, Tests) -> F = fun({N,P,Ws,E}, BadL) -> case catch run_test(Config, P, Ws) of diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 13c5662741..31ea3210a8 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -825,12 +825,13 @@ type_examples() -> %% is_subtype(V, T) syntax, we need a few examples of the syntax. {ex31,<<"-spec t1(FooBar :: t99()) -> t99();" "(t2()) -> t2();" - "('\\'t::4'()) -> '\\'t::4'() when is_subtype('\\'t::4'(), t24);" - "(t23()) -> t23() when is_subtype(t23(), atom())," - " is_subtype(t23(), t14());" - "(t24()) -> t24() when is_subtype(t24(), atom())," - " is_subtype(t24(), t14())," - " is_subtype(t24(), '\\'t::4'()).">>}, + "('\\'t::4'()) -> {'\\'t::4'(), B}" + " when is_subtype(B, '\\'t::4'());" + "(t23()) -> C when is_subtype(C, atom())," + " is_subtype(C, t14());" + "(t24()) -> D when is_subtype(D, atom())," + " is_subtype(D, t14())," + " is_subtype(D, '\\'t::4'()).">>}, {ex32,<<"-spec mod:t2() -> any(). ">>}, {ex33,<<"-opaque attributes_data() :: " "[{'column', column()} | {'line', info_line()} |" diff --git a/lib/xmerl/src/xmerl_scan.erl b/lib/xmerl/src/xmerl_scan.erl index 5e0459ec21..9f6b27113e 100644 --- a/lib/xmerl/src/xmerl_scan.erl +++ b/lib/xmerl/src/xmerl_scan.erl @@ -2225,16 +2225,18 @@ processed_whole_element(S=#xmerl_scanner{hook_fun = _Hook, AllAttrs = case S#xmerl_scanner.default_attrs of true -> - [ #xmlAttribute{name = AttName, - parents = [{Name, Pos} | Parents], - language = Lang, - nsinfo = NSI, - namespace = Namespace, - value = AttValue, - normalized = true} || - {AttName, AttValue} <- get_default_attrs(S, Name), - AttValue =/= no_value, - not lists:keymember(AttName, #xmlAttribute.name, Attrs) ]; + DefaultAttrs = + [ #xmlAttribute{name = AttName, + parents = [{Name, Pos} | Parents], + language = Lang, + nsinfo = NSI, + namespace = Namespace, + value = AttValue, + normalized = true} || + {AttName, AttValue} <- get_default_attrs(S, Name), + AttValue =/= no_value, + not lists:keymember(AttName, #xmlAttribute.name, Attrs) ], + lists:append(Attrs, DefaultAttrs); false -> Attrs end, diff --git a/lib/xmerl/test/xmerl_SUITE.erl b/lib/xmerl/test/xmerl_SUITE.erl index e97b8c6a4b..cf7c0b7548 100644 --- a/lib/xmerl/test/xmerl_SUITE.erl +++ b/lib/xmerl/test/xmerl_SUITE.erl @@ -54,7 +54,8 @@ groups() -> cpd_expl_provided_DTD]}, {misc, [], [latin1_alias, syntax_bug1, syntax_bug2, syntax_bug3, - pe_ref1, copyright, testXSEIF, export_simple1, export]}, + pe_ref1, copyright, testXSEIF, export_simple1, export, + default_attrs_bug]}, {eventp_tests, [], [sax_parse_and_export]}, {ticket_tests, [], [ticket_5998, ticket_7211, ticket_7214, ticket_7430, @@ -223,6 +224,21 @@ syntax_bug3(Config) -> Err -> Err end. +default_attrs_bug(Config) -> + file:set_cwd(datadir(Config)), + Doc = "<!DOCTYPE doc [<!ATTLIST doc b CDATA \"default\">]>\n" + "<doc a=\"explicit\"/>", + {#xmlElement{attributes = [#xmlAttribute{name = a, value = "explicit"}, + #xmlAttribute{name = b, value = "default"}]}, + [] + } = xmerl_scan:string(Doc, [{default_attrs, true}]), + Doc2 = "<!DOCTYPE doc [<!ATTLIST doc b CDATA \"default\">]>\n" + "<doc b=\"also explicit\" a=\"explicit\"/>", + {#xmlElement{attributes = [#xmlAttribute{name = b, value = "also explicit"}, + #xmlAttribute{name = a, value = "explicit"}]}, + [] + } = xmerl_scan:string(Doc2, [{default_attrs, true}]). + pe_ref1(Config) -> file:set_cwd(datadir(Config)), {#xmlElement{},[]} = xmerl_scan:file(datadir_join(Config,[misc,"PE_ref1.xml"]),[{validation,true}]). diff --git a/otp_versions.table b/otp_versions.table index e21ca7fdf4..575a358010 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-19.2.1 : erts-8.2.1 # asn1-4.0.4 common_test-1.13 compiler-7.0.3 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.2 debugger-4.2.1 dialyzer-3.0.3 diameter-1.12.1 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.2 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.3 ic-4.4.2 inets-6.3.4 jinterface-1.7.1 kernel-5.1.1 megaco-3.18.1 mnesia-4.14.2 observer-2.3 odbc-2.12 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.3 reltool-0.7.2 runtime_tools-1.11 sasl-3.0.2 snmp-5.2.4 ssh-4.4 ssl-8.1 stdlib-3.2 syntax_tools-2.1.1 tools-2.9 typer-0.9.11 wx-1.8 xmerl-1.3.12 : OTP-19.2 : common_test-1.13 compiler-7.0.3 crypto-3.7.2 dialyzer-3.0.3 edoc-0.8.1 erl_docgen-0.6.1 erl_interface-3.9.2 erts-8.2 eunit-2.3.2 hipe-3.15.3 inets-6.3.4 kernel-5.1.1 mnesia-4.14.2 observer-2.3 odbc-2.12 parsetools-2.1.4 public_key-1.3 runtime_tools-1.11 sasl-3.0.2 ssh-4.4 ssl-8.1 stdlib-3.2 syntax_tools-2.1.1 tools-2.9 wx-1.8 # asn1-4.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 debugger-4.2.1 diameter-1.12.1 eldap-1.2.2 et-1.6 gs-1.6.2 ic-4.4.2 jinterface-1.7.1 megaco-3.18.1 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 percept-0.9 reltool-0.7.2 snmp-5.2.4 typer-0.9.11 xmerl-1.3.12 : OTP-19.1.6 : erts-8.1.1 # asn1-4.0.4 common_test-1.12.3 compiler-7.0.2 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.1 debugger-4.2.1 dialyzer-3.0.2 diameter-1.12.1 edoc-0.8 eldap-1.2.2 erl_docgen-0.6 erl_interface-3.9.1 et-1.6 eunit-2.3.1 gs-1.6.2 hipe-3.15.2 ic-4.4.2 inets-6.3.3 jinterface-1.7.1 kernel-5.1 megaco-3.18.1 mnesia-4.14.1 observer-2.2.2 odbc-2.11.3 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.3 percept-0.9 public_key-1.2 reltool-0.7.2 runtime_tools-1.10.1 sasl-3.0.1 snmp-5.2.4 ssh-4.3.6 ssl-8.0.3 stdlib-3.1 syntax_tools-2.1 tools-2.8.6 typer-0.9.11 wx-1.7.1 xmerl-1.3.12 : OTP-19.1.5 : ssh-4.3.6 # asn1-4.0.4 common_test-1.12.3 compiler-7.0.2 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.1 debugger-4.2.1 dialyzer-3.0.2 diameter-1.12.1 edoc-0.8 eldap-1.2.2 erl_docgen-0.6 erl_interface-3.9.1 erts-8.1 et-1.6 eunit-2.3.1 gs-1.6.2 hipe-3.15.2 ic-4.4.2 inets-6.3.3 jinterface-1.7.1 kernel-5.1 megaco-3.18.1 mnesia-4.14.1 observer-2.2.2 odbc-2.11.3 orber-3.8.2 os_mon-2.4.1 otp_mibs-1.1.1 parsetools-2.1.3 percept-0.9 public_key-1.2 reltool-0.7.2 runtime_tools-1.10.1 sasl-3.0.1 snmp-5.2.4 ssl-8.0.3 stdlib-3.1 syntax_tools-2.1 tools-2.8.6 typer-0.9.11 wx-1.7.1 xmerl-1.3.12 : |