diff options
Diffstat (limited to 'lib')
122 files changed, 7250 insertions, 1490 deletions
diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index 9ec0d93e93..9eec05abd1 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -1710,7 +1710,7 @@ check_value(S,#valuedef{pos=Pos,name=Name,type=Type, {valueset, check_type(S,#typedef{pos=Pos,name=Name,typespec=NewType},NewType)}; check_value(S, #valuedef{}=V) -> - ?dbg("check_value, V: ~p~n",[V0]), + ?dbg("check_value, V: ~p~n",[V]), case V of #valuedef{checked=true} -> V; @@ -1721,7 +1721,8 @@ check_value(S, #valuedef{}=V) -> check_valuedef(#state{recordtopname=TopName}=S0, V0) -> #valuedef{name=Name,type=Vtype0,value=Value,module=ModName} = V0, V = V0#valuedef{checked=true}, - Vtype = check_type(S0, #typedef{name=Name,typespec=Vtype0},Vtype0), + Vtype1 = expand_valuedef_type(Vtype0), + Vtype = check_type(S0, #typedef{name=Name,typespec=Vtype1},Vtype1), Def = Vtype#type.def, S1 = S0#state{tname=Def}, SVal = update_state(S1, ModName), @@ -1767,6 +1768,27 @@ check_valuedef(#state{recordtopname=TopName}=S0, V0) -> V#valuedef{value=normalize_value(SVal, Vtype, Value, TopName)} end. +expand_valuedef_type(#type{def=Seq}=Type) + when is_record(Seq,'SEQUENCE') -> + NewComponents = case Seq#'SEQUENCE'.components of + {R1,_Ext,R2} -> R1 ++ R2; + {Root,_Ext} -> Root; + Root -> take_only_rootset(Root) + end, + NewSeq = Seq#'SEQUENCE'{components = NewComponents}, + Type#type{def=NewSeq}; +expand_valuedef_type(#type{def=Set}=Type) + when is_record(Set,'SET') -> + NewComponents = case Set#'SET'.components of + {R1,_Ext,R2} -> R1 ++ R2; + {Root,_Ext} -> Root; + Root -> take_only_rootset(Root) + end, + NewSet = Set#'SET'{components = NewComponents}, + Type#type{def=NewSet}; +expand_valuedef_type(Type) -> + Type. + is_contextswitchtype(#typedef{name='EXTERNAL'})-> true; is_contextswitchtype(#typedef{name='EMBEDDED PDV'}) -> @@ -1998,7 +2020,8 @@ normalize_value(S, Type, {'DEFAULT',Value}, NameList) -> {'ENUMERATED',CType,_} -> normalize_enumerated(S,Value,CType); {'CHOICE',CType,NewNameList} -> - normalize_choice(S,Value,CType,NewNameList); + ChoiceComponents = get_choice_components(S, {'CHOICE',CType}), + normalize_choice(S,Value,ChoiceComponents,NewNameList); {'SEQUENCE',CType,NewNameList} -> normalize_sequence(S,Value,CType,NewNameList); {'SEQUENCE OF',CType,NewNameList} -> @@ -2140,6 +2163,9 @@ normalize_octetstring(S, Value) -> _ -> asn1_error(S, illegal_octet_string_value) end; + Val when is_binary(Val) -> + %% constant default value + Val; _ -> asn1_error(S, illegal_octet_string_value) end. @@ -2751,8 +2777,9 @@ check_type(S=#state{recordtopname=TopName},Type,Ts) when is_record(Ts,type) -> TempNewDef#newt{type={'SEQUENCE OF',check_sequenceof(S,Type,Components)}, tag= merge_tags(Tag,?TAG_CONSTRUCTED(?N_SEQUENCE))}; - {'CHOICE',Components} -> + {'CHOICE',_} = Choice-> Ct = maybe_illicit_implicit_tag(S, choice, Tag), + Components = get_choice_components(S, Choice), TempNewDef#newt{type={'CHOICE',check_choice(S,Type,Components)},tag=Ct}; Set when is_record(Set,'SET') -> RecordName= diff --git a/lib/asn1/test/Makefile b/lib/asn1/test/Makefile index c38d1c6ebd..6ff4aa8d0f 100644 --- a/lib/asn1/test/Makefile +++ b/lib/asn1/test/Makefile @@ -60,6 +60,7 @@ MODULES= \ testSeqOf \ testSeqOfIndefinite \ testSeqOfCho \ + testSeqOfChoExt \ testSeqOfExternal \ testSeqOfTag \ testSetDefault \ @@ -72,6 +73,7 @@ MODULES= \ testSetTypeRefPrim \ testSetTypeRefSeq \ testSetTypeRefSet \ + testDefaultOctetString \ testChoiceIndefinite \ testSetOf \ testSetOfCho \ diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index ab78678110..a88e464996 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -99,6 +99,7 @@ groups() -> testChoTypeRefPrim, testChoTypeRefSeq, testChoTypeRefSet, + testDefaultOctetString, testMultipleLevels, testOpt, testSeqDefault, @@ -118,6 +119,7 @@ groups() -> {group, [], [testSeqOf, testSeqOfIndefinite]}, % Uses 'Mvrasn*' testSeqOfCho, + testSeqOfChoExt, testSetDefault, testExtensionAdditionGroup, testSetOptional, @@ -430,6 +432,11 @@ testChoTypeRefSet(Config, Rule, Opts) -> asn1_test_lib:compile("ChoTypeRefSet", Config, [Rule|Opts]), testChoTypeRefSet:set(Rule). +testDefaultOctetString(Config) -> test(Config, fun testDefaultOctetString/3). +testDefaultOctetString(Config, Rule, Opts) -> + asn1_test_lib:compile("DefaultOctetString", Config, [Rule|Opts]), + testDefaultOctetString:dos(Rule). + testMultipleLevels(Config) -> test(Config, fun testMultipleLevels/3). testMultipleLevels(Config, Rule, Opts) -> asn1_test_lib:compile("MultipleLevels", Config, [Rule|Opts]), @@ -535,6 +542,11 @@ testSeqOfCho(Config, Rule, Opts) -> asn1_test_lib:compile("SeqOfCho", Config, [Rule|Opts]), testSeqOfCho:main(Rule). +testSeqOfChoExt(Config) -> test(Config, fun testSeqOfChoExt/3). +testSeqOfChoExt(Config, Rule, Opts) -> + asn1_test_lib:compile("SeqOfChoExt", Config, [Rule|Opts]), + testSeqOfChoExt:main(Rule). + testSeqOfIndefinite(Config) -> test(Config, fun testSeqOfIndefinite/3, [ber]). testSeqOfIndefinite(Config, Rule, Opts) -> diff --git a/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 b/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 index 18473bae30..c488704196 100644 --- a/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ChoExtension.asn1 @@ -41,4 +41,6 @@ ChoExt4 ::= CHOICE str OCTET STRING } +choExt1 ChoExt1 ::= int : 1 + END diff --git a/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn b/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn new file mode 100644 index 0000000000..076e965d58 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/DefaultOctetString.asn @@ -0,0 +1,15 @@ +DefaultOctetString +DEFINITIONS +AUTOMATIC TAGS + ::= +BEGIN +Dos ::= SEQUENCE { + opt [2] OCTET STRING (SIZE(2..4)) OPTIONAL, + def [10] OCTET STRING (SIZE (1)) DEFAULT '05'H +} + +dos Dos ::= { + opt '1234'H +} + +END diff --git a/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 b/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 new file mode 100644 index 0000000000..51077754fd --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/SeqOfChoExt.asn1 @@ -0,0 +1,27 @@ +SeqOfChoExt DEFINITIONS AUTOMATIC TAGS EXTENSIBILITY IMPLIED ::=
+BEGIN
+
+Seq2 ::= SEQUENCE {
+ octstr [PRIVATE 6] OCTET STRING OPTIONAL
+}
+
+SeqOfCho ::= SEQUENCE OF CHOICE {
+ nullable NULL,
+ seq2 Seq2
+}
+
+Seq1 ::= SEQUENCE {
+ int INTEGER,
+ soc SeqOfCho
+}
+
+seq1 Seq1 ::= {
+ int 10,
+ soc {
+ seq2 : {
+ octstr '01020A'H
+ }
+ }
+}
+
+END
diff --git a/lib/asn1/test/testChoExtension.erl b/lib/asn1/test/testChoExtension.erl index 4c632aab81..cfb28be5c7 100644 --- a/lib/asn1/test/testChoExtension.erl +++ b/lib/asn1/test/testChoExtension.erl @@ -28,6 +28,7 @@ extension(_Rules) -> roundtrip('ChoExt1', {bool,true}), roundtrip('ChoExt1', {int,33}), + {int, 1} = 'ChoExtension':choExt1(), %% A trick to encode with another compatible CHOICE type to test reception %% extension alternative diff --git a/lib/asn1/test/testDefaultOctetString.erl b/lib/asn1/test/testDefaultOctetString.erl new file mode 100644 index 0000000000..82cd5810e5 --- /dev/null +++ b/lib/asn1/test/testDefaultOctetString.erl @@ -0,0 +1,34 @@ +-module(testDefaultOctetString). + +-export([dos/1]). + +-include_lib("common_test/include/ct.hrl"). + +-record('Dos', { + opt = asn1_NOVALUE, + def = asn1_DEFAULT +}). + +-define(def_DEFAULT, <<5>>). + +dos(Rules) -> + %% test roundtrip default + E1 = roundtrip(#'Dos'{}, #'Dos'{def = ?def_DEFAULT}), + %% test the value dos defined in the .asn file + E2 = roundtrip('DefaultOctetString':dos()), + %% sanity test a fully specified SEQUENCE + E3 = roundtrip(#'Dos'{opt = <<1,2,3>>, def = <<6>>}), + %% test def is/isn't encoded according to the value + if Rules == ber -> + <<48, 0>> = E1, + <<48, 4, 16#82, 2, 16#12, 16#34>> = E2, + <<48, 8, 16#82, 3, 1, 2, 3, 16#8A, 1, 6>> = E3; + true -> + ignore + end, + ok. + +roundtrip(Value) -> + roundtrip(Value, Value). +roundtrip(Value, Exp) -> + asn1_test_lib:roundtrip('DefaultOctetString', 'Dos', Value, Exp). diff --git a/lib/asn1/test/testSeqOfChoExt.erl b/lib/asn1/test/testSeqOfChoExt.erl new file mode 100644 index 0000000000..1e72c60841 --- /dev/null +++ b/lib/asn1/test/testSeqOfChoExt.erl @@ -0,0 +1,15 @@ +-module(testSeqOfChoExt). + +-export([main/1]). + +%-record('Seq2', { octstr = asn1_NOVALUE }). +%-record('Seq1', { int, soc }). + +main(_Rules) -> + roundtrip('SeqOfChoExt':seq1()). + +roundtrip(Value) -> + roundtrip(Value, Value). +roundtrip(Value, Exp) -> + Type = element(1,Value), + asn1_test_lib:roundtrip('SeqOfChoExt', Type, Value, Exp). diff --git a/lib/common_test/test_server/configure.in b/lib/common_test/test_server/configure.in index e07bd4c2aa..a32d050081 100644 --- a/lib/common_test/test_server/configure.in +++ b/lib/common_test/test_server/configure.in @@ -171,7 +171,7 @@ case $system in fi SHLIB_EXTRACT_ALL="" ;; - *-openbsd*) + *-openbsd*|*-netbsd*|*-freebsd*|*-dragonfly*) # Not available on all versions: check for include file. AC_CHECK_HEADER(dlfcn.h, [ SHLIB_CFLAGS="-fpic" @@ -194,29 +194,6 @@ case $system in ]) SHLIB_EXTRACT_ALL="" ;; - *-netbsd*|*-freebsd*|*-dragonfly*) - # Not available on all versions: check for include file. - AC_CHECK_HEADER(dlfcn.h, [ - SHLIB_CFLAGS="-fpic" - SHLIB_LD="ld" - SHLIB_LDFLAGS="$LDFLAGS -Bshareable -x" - SHLIB_SUFFIX=".so" - if test X${enable_m64_build} = Xyes; then - AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) - fi - if test X${enable_m32_build} = Xyes; then - AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) - fi - ], [ - # No dynamic loading. - SHLIB_CFLAGS="" - SHLIB_LD="ld" - SHLIB_LDFLAGS="" - SHLIB_SUFFIX="" - AC_MSG_ERROR(don't know how to compile and link dynamic drivers) - ]) - SHLIB_EXTRACT_ALL="" - ;; *-solaris2*|*-sysv4*) SHLIB_CFLAGS="-KPIC" SHLIB_LD="/usr/ccs/bin/ld" diff --git a/lib/compiler/scripts/smoke-mix.exs b/lib/compiler/scripts/smoke-mix.exs index 82ae3370fe..ba0815e465 100644 --- a/lib/compiler/scripts/smoke-mix.exs +++ b/lib/compiler/scripts/smoke-mix.exs @@ -25,6 +25,14 @@ defmodule Smoke.MixProject do [ {:bear, "~> 0.8.7"}, {:cloudi_core, "~> 1.7"}, + {:cloudi_service_monitoring, "~> 1.7"}, + {:cloudi_service_tcp, "~> 1.7"}, + {:cloudi_service_queue, "~> 1.7"}, + {:cloudi_service_udp, "~> 1.7"}, + {:cloudi_service_map_reduce, "~> 1.7"}, + {:cloudi_service_api_requests, "~> 1.7"}, + {:cloudi_service_router, "~> 1.7"}, + {:cloudi_service_request_rate, "~> 1.7"}, {:concuerror, "~> 0.20.0"}, {:cowboy, "~> 2.6.1"}, {:ecto, "~> 3.0.6"}, diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index c971e8844d..9f8d63baa1 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -129,9 +129,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif ERL_COMPILE_FLAGS += +inline +warn_unused_import \ - -Werror \ -I../../stdlib/include -I$(EGEN) -W +warn_missing_spec # ---------------------------------------------------- diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index c01ea4af91..06b42f1928 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -267,10 +267,29 @@ opt_is([#b_set{op=call,args=Args0,dst=Dst}=I0|Is], I1 = beam_ssa:normalize(I0#b_set{args=Args}), {Ts1,Ds,Fdb,I2} = opt_call(I1, D, Ts0, Ds0, Fdb0), case {map_get(Dst, Ts1),Is} of - {_,[#b_set{op=succeeded}]} -> + {Type,[#b_set{op=succeeded}]} when Type =/= none -> %% This call instruction is inside a try/catch - %% block. Don't attempt to optimize it. + %% block. Don't attempt to simplify it. opt_is(Is, Ts1, Ds, Fdb, D, Sub0, [I2|Acc]); + {none,[#b_set{op=succeeded}]} -> + %% This call instruction is inside a try/catch + %% block, but we know it will never return and + %% later optimizations may try to exploit that. + %% + %% For example, if we have an expression that + %% either returns this call or a tuple, we know + %% that the expression always returns a tuple + %% and can turn a later element/3 into + %% get_tuple_element. + %% + %% This is sound but difficult to validate in a + %% meaningful way as try/catch currently forces + %% us to maintain the illusion that the success + %% block is reachable even when its not, so we + %% disable the optimization to keep things + %% simple. + Ts = Ts1#{ Dst := any }, + opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I2|Acc]); {none,_} -> %% This call never returns. The rest of the %% instructions will not be executed. diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 4fba3fa1c6..efd2be94cb 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -2899,8 +2899,6 @@ lists_mod_return_type(filter, 2, _Vst) -> list; lists_mod_return_type(flatten, 1, _Vst) -> list; -lists_mod_return_type(flatten, 2, _Vst) -> - list; lists_mod_return_type(map, 2, Vst) -> same_length_type({x,1}, Vst); lists_mod_return_type(MF, 3, Vst) when MF =:= mapfoldl; MF =:= mapfoldr -> @@ -2912,8 +2910,6 @@ lists_mod_return_type(reverse, 1, Vst) -> same_length_type({x,0}, Vst); lists_mod_return_type(seq, 2, _Vst) -> list; -lists_mod_return_type(seq, 3, _Vst) -> - list; lists_mod_return_type(sort, 1, Vst) -> same_length_type({x,0}, Vst); lists_mod_return_type(sort, 2, Vst) -> @@ -2927,16 +2923,10 @@ lists_mod_return_type(unzip, 1, Vst) -> two_tuple(ListType, ListType); lists_mod_return_type(usort, 1, Vst) -> same_length_type({x,0}, Vst); -lists_mod_return_type(usort, 2, Vst) -> - same_length_type({x,1}, Vst); lists_mod_return_type(zip, 2, _Vst) -> list; -lists_mod_return_type(zip3, 3, _Vst) -> - list; lists_mod_return_type(zipwith, 3, _Vst) -> list; -lists_mod_return_type(zipwith3, 4, _Vst) -> - list; lists_mod_return_type(_, _, _) -> term. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 11dea9524b..28db8986ff 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -290,6 +290,10 @@ format_error(bad_crypto_key) -> "invalid crypto key."; format_error(no_crypto_key) -> "no crypto key supplied."; +format_error({unimplemented_instruction,Instruction}) -> + io_lib:fwrite("native-code compilation failed because of an " + "unimplemented instruction (~s).", + [Instruction]); format_error({native, E}) -> io_lib:fwrite("native-code compilation failed with reason: ~tP.", [E, 25]); @@ -1651,18 +1655,22 @@ native_compile_1(Code, St) -> case IgnoreErrors of true -> Ws = [{St#compile.ifile,[{none,?MODULE,{native,R}}]}], - {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> Es = [{St#compile.ifile,[{none,?MODULE,{native,R}}]}], {error,St#compile{errors=St#compile.errors ++ Es}} end catch + exit:{unimplemented_instruction,_}=Unimplemented -> + Ws = [{St#compile.ifile, + [{none,?MODULE,Unimplemented}]}], + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; Class:R:Stack -> case IgnoreErrors of true -> Ws = [{St#compile.ifile, [{none,?MODULE,{native_crash,R,Stack}}]}], - {ok,St#compile{warnings=St#compile.warnings ++ Ws}}; + {ok,Code,St#compile{warnings=St#compile.warnings ++ Ws}}; false -> erlang:raise(Class, R, Stack) end diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 7e219da0af..4939a94a92 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -1997,53 +1997,22 @@ case_opt_compiler_generated(Core) -> %% case_expand_var(Expr0, Sub) -> Expr -%% If Expr0 is a variable that has been previously matched and -%% is known to be a tuple, return the tuple instead. Otherwise +%% If Expr0 is a variable that is known to be bound to a +%% constructed tuple, return the tuple instead. Otherwise %% return Expr0 unchanged. -%% + case_expand_var(E, #sub{t=Tdb}) -> Key = cerl:var_name(E), case Tdb of - #{Key:=T0} -> - case cerl:is_c_tuple(T0) of - false -> - E; - true -> - %% The pattern was a tuple. Now we must make sure - %% that the elements of the tuple are suitable. In - %% particular, we don't want binary or map - %% construction here, since that means that the - %% binary or map will be constructed in the 'case' - %% argument. That is wasteful for binaries. Even - %% worse is that any map pattern that use the ':=' - %% operator will fail when used in map - %% construction (only the '=>' operator is allowed - %% when constructing a map from scratch). - try - cerl_trees:map(fun coerce_to_data/1, T0) - catch - throw:impossible -> - %% Something unsuitable was found (map or - %% or binary). Keep the variable. - E - end + #{Key:=T} -> + case cerl:is_c_tuple(T) of + false -> E; + true -> T end; _ -> E end. -%% coerce_to_data(Core) -> Core' -%% Coerce an element originally from a pattern to an data item or or -%% variable. Throw an 'impossible' exception if non-data Core Erlang -%% terms such as binary construction or map construction are -%% encountered. - -coerce_to_data(C) -> - case cerl:is_data(C) orelse cerl:is_c_var(C) of - true -> C; - false -> throw(impossible) - end. - %% case_opt_nomatch(E, Clauses, LitExpr) -> Clauses' %% Remove all clauses that cannot possibly match. diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl index a7ffc3f60a..882e281a44 100644 --- a/lib/compiler/test/beam_type_SUITE.erl +++ b/lib/compiler/test/beam_type_SUITE.erl @@ -24,7 +24,7 @@ integers/1,numbers/1,coverage/1,booleans/1,setelement/1, cons/1,tuple/1,record_float/1,binary_float/1,float_compare/1, arity_checks/1,elixir_binaries/1,find_best/1, - test_size/1]). + test_size/1,cover_lists_functions/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -46,7 +46,8 @@ groups() -> arity_checks, elixir_binaries, find_best, - test_size + test_size, + cover_lists_functions ]}]. init_per_suite(Config) -> @@ -473,5 +474,18 @@ do_test_size(Term) when is_tuple(Term) -> do_test_size(Term) when is_binary(Term) -> size(Term). +cover_lists_functions(Config) -> + case lists:suffix([no|Config], Config) of + true -> + ct:fail(should_be_false); + false -> + ok + end, + Zipped = lists:zipwith(fun(A, B) -> {A,B} end, + lists:duplicate(length(Config), zip), + Config), + true = is_list(Zipped), + ok. + id(I) -> I. diff --git a/lib/compiler/test/bs_construct_SUITE.erl b/lib/compiler/test/bs_construct_SUITE.erl index 69017d87e7..bccd70d6cb 100644 --- a/lib/compiler/test/bs_construct_SUITE.erl +++ b/lib/compiler/test/bs_construct_SUITE.erl @@ -27,6 +27,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, + verify_highest_opcode/1, two/1,test1/1,fail/1,float_bin/1,in_guard/1,in_catch/1, nasty_literals/1,coerce_to_float/1,side_effect/1, opt/1,otp_7556/1,float_arith/1,otp_8054/1, @@ -43,7 +44,8 @@ all() -> groups() -> [{p,[parallel], - [two,test1,fail,float_bin,in_guard,in_catch, + [verify_highest_opcode, + two,test1,fail,float_bin,in_guard,in_catch, nasty_literals,side_effect,opt,otp_7556,float_arith, otp_8054,cover]}]. @@ -68,6 +70,20 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_construct_r21_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 163 -> + ok; + TooHigh -> + ct:fail({too_high_opcode_for_21,TooHigh}) + end; + _ -> + ok + end. + two(Config) when is_list(Config) -> <<0,1,2,3,4,6,7,8,9>> = two_1([0], [<<1,2,3,4>>,<<6,7,8,9>>]), ok. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 2cfcb841a7..41e4918b1e 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -24,6 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, + verify_highest_opcode/1, size_shadow/1,int_float/1,otp_5269/1,null_fields/1,wiger/1, bin_tail/1,save_restore/1, partitioned_bs_match/1,function_clause/1, @@ -60,7 +61,8 @@ all() -> groups() -> [{p,[], - [size_shadow,int_float,otp_5269,null_fields,wiger, + [verify_highest_opcode, + size_shadow,int_float,otp_5269,null_fields,wiger, bin_tail,save_restore, partitioned_bs_match,function_clause,unit, shared_sub_bins,bin_and_float,dec_subidentifiers, @@ -101,6 +103,20 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> end_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ok. +verify_highest_opcode(_Config) -> + case ?MODULE of + bs_match_r21_SUITE -> + {ok,Beam} = file:read_file(code:which(?MODULE)), + case test_lib:highest_opcode(Beam) of + Highest when Highest =< 163 -> + ok; + TooHigh -> + ct:fail({too_high_opcode_for_21,TooHigh}) + end; + _ -> + ok + end. + size_shadow(Config) when is_list(Config) -> %% Originally OTP-5270. 7 = size_shadow_1(), diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 408af80dd9..53627b9d81 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1431,9 +1431,7 @@ bc_options(Config) -> highest_opcode(DataDir, Mod, Opt) -> Src = filename:join(DataDir, atom_to_list(Mod)++".erl"), {ok,Mod,Beam} = compile:file(Src, [binary|Opt]), - {ok,{Mod,[{"Code",Code}]}} = beam_lib:chunks(Beam, ["Code"]), - <<16:32,0:32,HighestOpcode:32,_/binary>> = Code, - HighestOpcode. + test_lib:highest_opcode(Beam). deterministic_include(Config) when is_list(Config) -> DataDir = proplists:get_value(data_dir, Config), diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index 39c26c6142..3348c6e9ea 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -22,7 +22,8 @@ -include_lib("common_test/include/ct.hrl"). -compile({no_auto_import,[binary_part/2]}). -export([id/1,recompile/1,parallel/0,uniq/0,opt_opts/1,get_data_dir/1, - is_cloned_mod/1,smoke_disasm/1,p_run/2]). + is_cloned_mod/1,smoke_disasm/1,p_run/2, + highest_opcode/1]). %% Used by test case that override BIFs. -export([binary_part/2,binary/1]). @@ -113,6 +114,14 @@ is_cloned_mod_1("_no_module_opt_SUITE") -> true; is_cloned_mod_1([_|T]) -> is_cloned_mod_1(T); is_cloned_mod_1([]) -> false. +%% Return the highest opcode use in the BEAM module. + +highest_opcode(Beam) -> + {ok,{_Mod,[{"Code",Code}]}} = beam_lib:chunks(Beam, ["Code"]), + FormatNumber = 0, + <<16:32,FormatNumber:32,HighestOpcode:32,_/binary>> = Code, + HighestOpcode. + %% p_run(fun(Data) -> ok|error, List) -> ok %% Will fail the test case if there were any errors. diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl index 8f9cd9ab1e..539f9d69fa 100644 --- a/lib/compiler/test/trycatch_SUITE.erl +++ b/lib/compiler/test/trycatch_SUITE.erl @@ -27,7 +27,8 @@ nested_horrid/1,last_call_optimization/1,bool/1, plain_catch_coverage/1,andalso_orelse/1,get_in_try/1, hockey/1,handle_info/1,catch_in_catch/1,grab_bag/1, - stacktrace/1,nested_stacktrace/1,raise/1]). + stacktrace/1,nested_stacktrace/1,raise/1, + no_return_in_try_block/1]). -include_lib("common_test/include/ct.hrl"). @@ -43,7 +44,8 @@ groups() -> nested_after,nested_horrid,last_call_optimization, bool,plain_catch_coverage,andalso_orelse,get_in_try, hockey,handle_info,catch_in_catch,grab_bag, - stacktrace,nested_stacktrace,raise]}]. + stacktrace,nested_stacktrace,raise, + no_return_in_try_block]}]. init_per_suite(Config) -> @@ -1287,5 +1289,26 @@ do_test_raise_4(Expr) -> erlang:raise(exit, {exception,C,E,Stk}, Stk) end. +no_return_in_try_block(Config) when is_list(Config) -> + 1.0 = no_return_in_try_block_1(0), + 1.0 = no_return_in_try_block_1(0.0), + + gurka = no_return_in_try_block_1(gurka), + [] = no_return_in_try_block_1([]), + + ok. + +no_return_in_try_block_1(H) -> + try + Float = if + is_number(H) -> float(H); + true -> no_return() + end, + Float + 1 + catch + throw:no_return -> H + end. + +no_return() -> throw(no_return). id(I) -> I. diff --git a/lib/crypto/c_src/aead.c b/lib/crypto/c_src/aead.c index 3ee04f1be9..4ed16615a5 100644 --- a/lib/crypto/c_src/aead.c +++ b/lib/crypto/c_src/aead.c @@ -39,87 +39,79 @@ ERL_NIF_TERM aead_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(argc == 6); if (!enif_is_atom(env, type)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-atom cipher type"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary key"); goto done;} if (!enif_inspect_binary(env, argv[2], &iv)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary iv"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[3], &aad)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary AAD"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[4], &in)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary text"); goto done;} if (!enif_get_uint(env, argv[5], &tag_len)) - goto bad_arg; + {ret = EXCP_BADARG(env, ""); goto done;} if (tag_len > INT_MAX || iv.size > INT_MAX || in.size > INT_MAX || aad.size > INT_MAX) - goto bad_arg; + {ret = EXCP_BADARG(env, "binary too long"); goto done;} if ((cipherp = get_cipher_type(type, key.size)) == NULL) - goto bad_arg; + {ret = EXCP_BADARG(env, "Unknown cipher"); goto done;} if (cipherp->flags & NON_EVP_CIPHER) - goto bad_arg; + {ret = EXCP_BADARG(env, "Bad cipher"); goto done;} if (! (cipherp->flags & AEAD_CIPHER) ) - goto bad_arg; + {ret = EXCP_BADARG(env, "Not aead cipher"); goto done;} if ((cipher = cipherp->cipher.p) == NULL) - return enif_raise_exception(env, atom_notsup); + {ret = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); goto done;} ctx_ctrl_set_ivlen = cipherp->extra.aead.ctx_ctrl_set_ivlen; ctx_ctrl_get_tag = cipherp->extra.aead.ctx_ctrl_get_tag; ctx_ctrl_set_tag = cipherp->extra.aead.ctx_ctrl_set_tag; if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_ivlen, (int)iv.size, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} #if defined(HAVE_CCM) if (type == atom_aes_ccm) { if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, (int)tag_len, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } else #endif { if (EVP_EncryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } if (EVP_EncryptUpdate(ctx, NULL, &len, aad.data, (int)aad.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if ((outp = enif_make_new_binary(env, in.size, &out)) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptUpdate(ctx, outp, &len, in.data, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_EncryptFinal_ex(ctx, outp/*+len*/, &len) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if ((tagp = enif_make_new_binary(env, tag_len, &out_tag)) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_get_tag, (int)tag_len, tagp) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} CONSUME_REDS(env, in); ret = enif_make_tuple2(env, out, out_tag); - goto done; - - bad_arg: - ret = enif_make_badarg(env); - goto done; - - err: - ret = atom_error; done: if (ctx) @@ -127,7 +119,7 @@ ERL_NIF_TERM aead_encrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return ret; #else - return enif_raise_exception(env, atom_notsup); + return EXCP_NOTSUP(env, ""); #endif } @@ -151,72 +143,72 @@ ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) #endif if (!enif_is_atom(env, type)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-atom cipher type"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[1], &key)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary key"); goto done;} if (!enif_inspect_binary(env, argv[2], &iv)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary iv"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[3], &aad)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary AAD"); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[4], &in)) - goto bad_arg; + {ret = EXCP_BADARG(env, ""); goto done;} if (!enif_inspect_iolist_as_binary(env, argv[5], &tag)) - goto bad_arg; + {ret = EXCP_BADARG(env, "non-binary text"); goto done;} if (tag.size > INT_MAX || key.size > INT_MAX || iv.size > INT_MAX || in.size > INT_MAX || aad.size > INT_MAX) - goto bad_arg; + {ret = EXCP_BADARG(env, "binary too long"); goto done;} if ((cipherp = get_cipher_type(type, key.size)) == NULL) - goto bad_arg; + {ret = EXCP_BADARG(env, "Unknown cipher"); goto done;} if (cipherp->flags & NON_EVP_CIPHER) - goto bad_arg; + {ret = EXCP_BADARG(env, "Bad cipher"); goto done;} if ( !(cipherp->flags & AEAD_CIPHER) ) - goto bad_arg; + {ret = EXCP_BADARG(env, "Not aead cipher"); goto done;} if ((cipher = cipherp->cipher.p) == NULL) - return enif_raise_exception(env, atom_notsup); + {ret = EXCP_NOTSUP(env, "Cipher not supported in this libcrypto version"); goto done;} ctx_ctrl_set_ivlen = cipherp->extra.aead.ctx_ctrl_set_ivlen; ctx_ctrl_set_tag = cipherp->extra.aead.ctx_ctrl_set_tag; if ((outp = enif_make_new_binary(env, in.size, &out)) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_ivlen, (int)iv.size, NULL) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} #if defined(HAVE_CCM) if (type == atom_aes_ccm) { if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, (int)tag.size, tag.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } else #endif { if (EVP_DecryptInit_ex(ctx, NULL, NULL, key.data, iv.data) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} } if (EVP_DecryptUpdate(ctx, NULL, &len, aad.data, (int)aad.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} if (EVP_DecryptUpdate(ctx, outp, &len, in.data, (int)in.size) != 1) - goto err; + {ret = EXCP_ERROR(env, ""); goto done;} #if defined(HAVE_GCM) if (type == atom_aes_gcm) { if (EVP_CIPHER_CTX_ctrl(ctx, ctx_ctrl_set_tag, (int)tag.size, tag.data) != 1) - goto err; + goto err; if (EVP_DecryptFinal_ex(ctx, outp+len, &len) != 1) goto err; } @@ -225,11 +217,8 @@ ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ret = out; goto done; - bad_arg: - ret = enif_make_badarg(env); - goto done; - err: + /* Decrypt failed, that is, wrong tag */ ret = atom_error; done: @@ -238,6 +227,6 @@ ERL_NIF_TERM aead_decrypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return ret; #else - return enif_raise_exception(env, atom_notsup); + return EXCP_NOTSUP(env, ""); #endif } diff --git a/lib/crypto/c_src/algorithms.c b/lib/crypto/c_src/algorithms.c index 06cd109fc1..1d45ed9df2 100644 --- a/lib/crypto/c_src/algorithms.c +++ b/lib/crypto/c_src/algorithms.c @@ -68,9 +68,15 @@ void init_algorithms_types(ErlNifEnv* env) // Non-validated algorithms follow algo_hash_fips_cnt = algo_hash_cnt; +#ifdef HAVE_MD4 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md4"); +#endif +#ifdef HAVE_MD5 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "md5"); +#endif +#ifdef HAVE_RIPEMD160 algo_hash[algo_hash_cnt++] = enif_make_atom(env, "ripemd160"); +#endif algo_pubkey_cnt = 0; algo_pubkey[algo_pubkey_cnt++] = enif_make_atom(env, "rsa"); diff --git a/lib/crypto/c_src/api_ng.c b/lib/crypto/c_src/api_ng.c index 6a833a0984..5d063c3ae4 100644 --- a/lib/crypto/c_src/api_ng.c +++ b/lib/crypto/c_src/api_ng.c @@ -29,18 +29,6 @@ ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM ng_crypto_one_shot(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - - -/* All nif functions return a valid value or throws an exception */ -#define EXCP(Env, Class, Str) enif_raise_exception((Env), \ - enif_make_tuple2((Env), (Class), \ - enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) - -#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) -#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) -#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) - - #ifdef HAVE_ECB_IVEC_BUG /* <= 0.9.8l returns faulty ivec length */ # define GET_IV_LEN(Ciph) ((Ciph)->flags & ECB_BUG_0_9_8L) ? 0 : EVP_CIPHER_iv_length((Ciph)->cipher.p) @@ -207,7 +195,7 @@ static int get_init_args(ErlNifEnv* env, goto err; } - +#ifdef HAVE_RC2 if (EVP_CIPHER_type((*cipherp)->cipher.p) == NID_rc2_cbc) { if (key_bin.size > INT_MAX / 8) { *return_term = EXCP_BADARG(env, "To large rc2_cbc key"); @@ -218,6 +206,7 @@ static int get_init_args(ErlNifEnv* env, goto err; } } +#endif if (ivec_arg == atom_undefined || ivec_len == 0) { @@ -346,7 +335,7 @@ ERL_NIF_TERM ng_crypto_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg ret = enif_make_resource(env, ctx_res); if(ctx_res) enif_release_resource(ctx_res); - } else if (enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) { + } else if (enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) { /* Fetch the flag telling if we are going to encrypt (=true) or decrypt (=false) */ if (argv[3] == atom_true) encflg = 1; @@ -426,7 +415,7 @@ ERL_NIF_TERM ng_crypto_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ struct evp_cipher_ctx *ctx_res; ERL_NIF_TERM ret; - if (!enif_get_resource(env, argv[0], evp_cipher_ctx_rtype, (void**)&ctx_res)) + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)evp_cipher_ctx_rtype, (void**)&ctx_res)) return EXCP_BADARG(env, "Bad 1:st arg"); if (argc == 3) { diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index 114e3c1985..0793ffa6ca 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -52,6 +52,12 @@ ERL_NIF_TERM atom_ecb_mode; ERL_NIF_TERM atom_cbc_mode; ERL_NIF_TERM atom_cfb_mode; ERL_NIF_TERM atom_ofb_mode; +ERL_NIF_TERM atom_ctr_mode; +ERL_NIF_TERM atom_gcm_mode; +ERL_NIF_TERM atom_ccm_mode; +ERL_NIF_TERM atom_xts_mode; +ERL_NIF_TERM atom_wrap_mode; +ERL_NIF_TERM atom_ocb_mode; ERL_NIF_TERM atom_stream_cipher; #if defined(HAVE_EC) @@ -164,6 +170,12 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM atom_cbc_mode = enif_make_atom(env,"cbc_mode"); atom_cfb_mode = enif_make_atom(env,"cfb_mode"); atom_ofb_mode = enif_make_atom(env,"ofb_mode"); + atom_ctr_mode = enif_make_atom(env,"ctr_mode"); + atom_gcm_mode = enif_make_atom(env,"gcm_mode"); + atom_ccm_mode = enif_make_atom(env,"ccm_mode"); + atom_xts_mode = enif_make_atom(env,"xts_mode"); + atom_wrap_mode = enif_make_atom(env,"wrap_mode"); + atom_ocb_mode = enif_make_atom(env,"ocb_mode"); atom_stream_cipher = enif_make_atom(env,"stream_cipher"); #if defined(HAVE_EC) diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index fc46d838aa..24f6dc26fd 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -56,6 +56,12 @@ extern ERL_NIF_TERM atom_ecb_mode; extern ERL_NIF_TERM atom_cbc_mode; extern ERL_NIF_TERM atom_cfb_mode; extern ERL_NIF_TERM atom_ofb_mode; +extern ERL_NIF_TERM atom_ctr_mode; +extern ERL_NIF_TERM atom_gcm_mode; +extern ERL_NIF_TERM atom_ccm_mode; +extern ERL_NIF_TERM atom_xts_mode; +extern ERL_NIF_TERM atom_wrap_mode; +extern ERL_NIF_TERM atom_ocb_mode; extern ERL_NIF_TERM atom_stream_cipher; #if defined(HAVE_EC) diff --git a/lib/crypto/c_src/cipher.c b/lib/crypto/c_src/cipher.c index 5c57898c50..2652e1db4e 100644 --- a/lib/crypto/c_src/cipher.c +++ b/lib/crypto/c_src/cipher.c @@ -28,12 +28,12 @@ static struct cipher_type_t cipher_types[] = { -#ifndef OPENSSL_NO_RC2 +#ifdef HAVE_RC2 {{"rc2_cbc"}, {&EVP_rc2_cbc}, 0, NO_FIPS_CIPHER}, #else {{"rc2_cbc"}, {NULL}, 0, NO_FIPS_CIPHER}, #endif -#ifndef OPENSSL_NO_RC4 +#ifdef HAVE_RC4 {{"rc4"}, {&EVP_rc4}, 0, NO_FIPS_CIPHER}, #else {{"rc4"}, {NULL}, 0, NO_FIPS_CIPHER}, @@ -274,6 +274,42 @@ ERL_NIF_TERM cipher_info_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] ret_mode = atom_ofb_mode; break; +#ifdef EVP_CIPH_CTR_MODE + case EVP_CIPH_CTR_MODE: + ret_mode = atom_ctr_mode; + break; +#endif + +#ifdef EVP_CIPH_GCM_MODE + case EVP_CIPH_GCM_MODE: + ret_mode = atom_gcm_mode; + break; +#endif + +#ifdef EVP_CIPH_CCM_MODE + case EVP_CIPH_CCM_MODE: + ret_mode = atom_ccm_mode; + break; +#endif + +#ifdef EVP_CIPH_XTS_MODE + case EVP_CIPH_XTS_MODE: + ret_mode = atom_xts_mode; + break; +#endif + +#ifdef EVP_CIPH_WRAP_MODE + case EVP_CIPH_WRAP_MODE: + ret_mode = atom_wrap_mode; + break; +#endif + +#ifdef EVP_CIPH_OCB_MODE + case EVP_CIPH_OCB_MODE: + ret_mode = atom_ocb_mode; + break; +#endif + case EVP_CIPH_STREAM_CIPHER: ret_mode = atom_stream_cipher; break; diff --git a/lib/crypto/c_src/common.h b/lib/crypto/c_src/common.h index 2bc8bdd73c..0bf7f09f4f 100644 --- a/lib/crypto/c_src/common.h +++ b/lib/crypto/c_src/common.h @@ -35,4 +35,15 @@ #include "openssl_config.h" #include "atoms.h" + +/* All nif functions return a valid value or throws an exception */ +#define EXCP(Env, Id, Str) enif_raise_exception((Env), \ + enif_make_tuple2((Env), \ + (Id), \ + enif_make_string((Env),(Str),(ERL_NIF_LATIN1)) )) + +#define EXCP_NOTSUP(Env, Str) EXCP((Env), atom_notsup, (Str)) +#define EXCP_BADARG(Env, Str) EXCP((Env), atom_badarg, (Str)) +#define EXCP_ERROR(Env, Str) EXCP((Env), atom_error, (Str)) + #endif /* E_COMMON_H__ */ diff --git a/lib/crypto/c_src/digest.c b/lib/crypto/c_src/digest.c index fec286c000..c987a664d5 100644 --- a/lib/crypto/c_src/digest.c +++ b/lib/crypto/c_src/digest.c @@ -22,10 +22,32 @@ static struct digest_type_t digest_types[] = { - {{"md4"}, {&EVP_md4}}, - {{"md5"}, {&EVP_md5}}, - {{"ripemd160"}, {&EVP_ripemd160}}, + {{"md4"}, +#ifdef HAVE_MD4 + {&EVP_md4} +#else + {NULL} +#endif + }, + + {{"md5"}, +#ifdef HAVE_MD5 + {&EVP_md5} +#else + {NULL} +#endif + }, + + {{"ripemd160"}, +#ifdef HAVE_RIPEMD160 + {&EVP_ripemd160} +#else + {NULL} +#endif + }, + {{"sha"}, {&EVP_sha1}}, + {{"sha224"}, #ifdef HAVE_SHA224 {&EVP_sha224} @@ -33,6 +55,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha256"}, #ifdef HAVE_SHA256 {&EVP_sha256} @@ -40,6 +63,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha384"}, #ifdef HAVE_SHA384 {&EVP_sha384} @@ -47,6 +71,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha512"}, #ifdef HAVE_SHA512 {&EVP_sha512} @@ -54,6 +79,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_224"}, #ifdef HAVE_SHA3_224 {&EVP_sha3_224} @@ -61,6 +87,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_256"}, #ifdef HAVE_SHA3_256 {&EVP_sha3_256} @@ -68,6 +95,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_384"}, #ifdef HAVE_SHA3_384 {&EVP_sha3_384} @@ -75,6 +103,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"sha3_512"}, #ifdef HAVE_SHA3_512 {&EVP_sha3_512} @@ -82,6 +111,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"blake2b"}, #ifdef HAVE_BLAKE2 {&EVP_blake2b512} @@ -89,6 +119,7 @@ static struct digest_type_t digest_types[] = {NULL} #endif }, + {{"blake2s"}, #ifdef HAVE_BLAKE2 {&EVP_blake2s256} diff --git a/lib/crypto/c_src/engine.c b/lib/crypto/c_src/engine.c index 7ffbb9e70d..ea5d9a588f 100644 --- a/lib/crypto/c_src/engine.c +++ b/lib/crypto/c_src/engine.c @@ -106,15 +106,13 @@ int init_engine_ctx(ErlNifEnv *env) { (ErlNifResourceDtor*) engine_ctx_dtor, ERL_NIF_RT_CREATE|ERL_NIF_RT_TAKEOVER, NULL); - if (engine_ctx_rtype == NULL) - goto err; + if (engine_ctx_rtype == NULL) { + PRINTF_ERR0("CRYPTO: Could not open resource type 'ENGINE_CTX'"); + return 0; + } #endif return 1; - - err: - PRINTF_ERR0("CRYPTO: Could not open resource type 'ENGINE_CTX'"); - return 0; } ERL_NIF_TERM engine_by_id_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) diff --git a/lib/crypto/c_src/hash.c b/lib/crypto/c_src/hash.c index 0a9f64acef..9b79258585 100644 --- a/lib/crypto/c_src/hash.c +++ b/lib/crypto/c_src/hash.c @@ -21,9 +21,15 @@ #include "hash.h" #include "digest.h" -#define MD5_CTX_LEN (sizeof(MD5_CTX)) -#define MD4_CTX_LEN (sizeof(MD4_CTX)) -#define RIPEMD160_CTX_LEN (sizeof(RIPEMD160_CTX)) +#ifdef HAVE_MD5 +# define MD5_CTX_LEN (sizeof(MD5_CTX)) +#endif +#ifdef HAVE_MD4 +# define MD4_CTX_LEN (sizeof(MD4_CTX)) +#endif +#ifdef HAVE_RIPEMD160 +# define RIPEMD160_CTX_LEN (sizeof(RIPEMD160_CTX)) +#endif #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) struct evp_md_ctx { @@ -261,18 +267,24 @@ ERL_NIF_TERM hash_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) switch (EVP_MD_type(digp->md.p)) { +#ifdef HAVE_MD4 case NID_md4: ctx_size = MD4_CTX_LEN; ctx_init = (init_fun)(&MD4_Init); break; +#endif +#ifdef HAVE_MD5 case NID_md5: ctx_size = MD5_CTX_LEN; ctx_init = (init_fun)(&MD5_Init); break; +#endif +#ifdef HAVE_RIPEMD160 case NID_ripemd160: ctx_size = RIPEMD160_CTX_LEN; ctx_init = (init_fun)(&RIPEMD160_Init); break; +#endif case NID_sha1: ctx_size = sizeof(SHA_CTX); ctx_init = (init_fun)(&SHA1_Init); @@ -352,18 +364,24 @@ ERL_NIF_TERM hash_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] switch (EVP_MD_type(digp->md.p)) { +#ifdef HAVE_MD4 case NID_md4: ctx_size = MD4_CTX_LEN; ctx_update = (update_fun)(&MD4_Update); break; +#endif +#ifdef HAVE_MD5 case NID_md5: ctx_size = MD5_CTX_LEN; ctx_update = (update_fun)(&MD5_Update); break; +#endif +#ifdef HAVE_RIPEMD160 case NID_ripemd160: ctx_size = RIPEMD160_CTX_LEN; ctx_update = (update_fun)(&RIPEMD160_Update); break; +#endif case NID_sha1: ctx_size = sizeof(SHA_CTX); ctx_update = (update_fun)(&SHA1_Update); @@ -448,18 +466,24 @@ ERL_NIF_TERM hash_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) switch (EVP_MD_type(md)) { +#ifdef HAVE_MD4 case NID_md4: ctx_size = MD4_CTX_LEN; ctx_final = (final_fun)(&MD4_Final); break; +#endif +#ifdef HAVE_MD5 case NID_md5: ctx_size = MD5_CTX_LEN; ctx_final = (final_fun)(&MD5_Final); break; - case NID_ripemd160: +#endif +#ifdef HAVE_RIPEMD160 + case NID_ripemd160: ctx_size = RIPEMD160_CTX_LEN; ctx_final = (final_fun)(&RIPEMD160_Final); break; +#endif case NID_sha1: ctx_size = sizeof(SHA_CTX); ctx_final = (final_fun)(&SHA1_Final); diff --git a/lib/crypto/c_src/hmac.c b/lib/crypto/c_src/hmac.c index c41e50eb35..ff7005d75e 100644 --- a/lib/crypto/c_src/hmac.c +++ b/lib/crypto/c_src/hmac.c @@ -181,7 +181,7 @@ ERL_NIF_TERM hmac_update_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] ASSERT(argc == 2); - if (!enif_get_resource(env, argv[0], hmac_context_rtype, (void**)&obj)) + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)hmac_context_rtype, (void**)&obj)) goto bad_arg; if (!enif_inspect_iolist_as_binary(env, argv[1], &data)) goto bad_arg; @@ -224,7 +224,7 @@ ERL_NIF_TERM hmac_final_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(argc == 1 || argc == 2); - if (!enif_get_resource(env, argv[0], hmac_context_rtype, (void**)&obj)) + if (!enif_get_resource(env, argv[0], (ErlNifResourceType*)hmac_context_rtype, (void**)&obj)) goto bad_arg; if (argc == 2) { if (!enif_get_uint(env, argv[1], &req_len)) diff --git a/lib/crypto/c_src/openssl_config.h b/lib/crypto/c_src/openssl_config.h index 46868cb987..f926f8af13 100644 --- a/lib/crypto/c_src/openssl_config.h +++ b/lib/crypto/c_src/openssl_config.h @@ -166,6 +166,28 @@ # define HAVE_BLAKE2 #endif +#ifndef OPENSSL_NO_MD4 +# define HAVE_MD4 +#endif + +#ifndef OPENSSL_NO_MD5 +# define HAVE_MD5 +#endif + +#ifndef OPENSSL_NO_RC2 +# define HAVE_RC2 +#endif + +#ifndef OPENSSL_NO_RC4 +# define HAVE_RC4 +#endif + +#ifndef OPENSSL_NO_RMD160 +/* Note RMD160 vs RIPEMD160 */ +# define HAVE_RIPEMD160 +#endif + + #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(0,9,8,'o') \ && !defined(OPENSSL_NO_EC) \ && !defined(OPENSSL_NO_ECDH) \ @@ -192,7 +214,9 @@ # define HAVE_AEAD # define HAVE_GCM # define HAVE_CCM -# define HAVE_CMAC +# ifndef OPENSSL_NO_CMAC +# define HAVE_CMAC +# endif # if defined(RSA_PKCS1_OAEP_PADDING) # define HAVE_RSA_OAEP_PADDING # endif @@ -204,21 +228,27 @@ #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0) # ifndef HAS_LIBRESSL -# define HAVE_CHACHA20_POLY1305 +# if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305) +# define HAVE_CHACHA20_POLY1305 +# endif # define HAVE_RSA_OAEP_MD # endif #endif #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION(1,1,0,'d') # ifndef HAS_LIBRESSL -# define HAVE_CHACHA20 +# ifndef OPENSSL_NO_CHACHA +# define HAVE_CHACHA20 +# endif # endif #endif // OPENSSL_VERSION_NUMBER >= 1.1.1-pre8 #if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)-7) # ifndef HAS_LIBRESSL -# define HAVE_POLY1305 +# if !defined(OPENSSL_NO_POLY1305) +# define HAVE_POLY1305 +# endif # endif #endif diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 83e10c4c78..8a4fad67de 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -192,7 +192,16 @@ <datatypes> <datatype_title>Ciphers</datatype_title> <datatype> + <name name="cipher"/> <name name="stream_cipher"/> + <name name="block_cipher"/> + <desc> + <p>Ciphers known byt the CRYPTO application. Note that this list might be reduced if the + underlying libcrypto does not support all of them.</p> + </desc> + </datatype> + + <datatype> <name name="stream_cipher_iv"/> <name name="stream_cipher_no_iv"/> <desc> @@ -204,7 +213,7 @@ </datatype> <datatype> - <name name="block_cipher_with_iv"/> + <name name="block_cipher_iv"/> <name name="cbc_cipher"/> <name name="cfb_cipher"/> <desc> @@ -228,7 +237,7 @@ </datatype> <datatype> - <name name="block_cipher_without_iv"/> + <name name="block_cipher_no_iv"/> <name name="ecb_cipher"/> <desc> <p>Block ciphers without initialization vector for @@ -248,20 +257,16 @@ </desc> </datatype> - <datatype_title>Digests</datatype_title> + <datatype_title>Digests and hash</datatype_title> <datatype> - <name name="sha1"/> - <name name="sha2"/> - <name name="sha3"/> - <name name="blake2"/> + <name name="hash_algorithm"/> <desc> </desc> </datatype> <datatype> - <name name="compatibility_only_hash"/> + <name name="hmac_hash_algorithm"/> <desc> - <p>The <c>compatibility_only_hash()</c> algorithms are recommended only for compatibility with existing applications.</p> </desc> </datatype> @@ -283,6 +288,17 @@ </desc> </datatype> + <datatype> + <name name="sha1"/> + <name name="sha2"/> + <name name="sha3"/> + <name name="blake2"/> + <name name="compatibility_only_hash"/> + <desc> + <p>The <c>compatibility_only_hash()</c> algorithms are recommended only for compatibility with existing applications.</p> + </desc> + </datatype> + <datatype_title>Elliptic Curves</datatype_title> <datatype> <name name="ec_named_curve"/> @@ -537,6 +553,52 @@ </desc> </datatype> + <datatype_title>Error types</datatype_title> + + <datatype> + <name name="run_time_error"/> + <desc> + <p>The exception <c>error:badarg</c> signifies that one or more arguments are of wrong data type, + or are otherwise badly formed. + </p> + <p>The exception <c>error:notsup</c> signifies that the algorithm is known but is not supported + by current underlying libcrypto or explicitly disabled when building that. + </p> + <p>For a list of supported algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </datatype> + + <datatype> + <name name="descriptive_error"/> + <desc> + <p>This is a more developed variant of the older + <seealso marker="#type-run_time_error">run_time_error()</seealso>. + </p> + <p>It is like the older type an exception of the <c>error</c> class. In addition they contain + a descriptive text in English. That text is targeted to a developer. Examples are "Bad key size" + or "Cipher id is not an atom". + </p> + <p>The exceptions are:</p> + <taglist> + <tag><c>{badarg, Description::string()}</c></tag> + <item><p>Signifies that one or more arguments are of wrong data type or are otherwise badly formed.</p> + </item> + + <tag><c>{notsup, Description::string()}</c></tag> + <item><p>Signifies that the algorithm is known but is not supported by current underlying libcrypto + or explicitly disabled when building that one.</p> + </item> + + <tag><c>{error, Description::string()}</c></tag> + <item><p>An error condition that should not occur, for example a memory allocation failed or + the underlying cryptolib returned an error code, for example "Can't initialize context, step 1". + Thoose text usually needs searching the C-code to be understood.</p> + </item> + </taglist> + </desc> + </datatype> + </datatypes> <!--================ FUNCTIONS ================--> @@ -568,17 +630,18 @@ </func> <func> - <name since="OTP R16B01">block_encrypt(Type, Key, Ivec, PlainText) -> CipherText</name> - <name since="OTP R16B01">block_encrypt(AeadType, Key, Ivec, {AAD, PlainText}) -> {CipherText, CipherTag}</name> - <name since="OTP R16B01">block_encrypt(aes_gcm | aes_ccm, Key, Ivec, {AAD, PlainText, TagLength}) -> {CipherText, CipherTag}</name> + <name since="OTP R16B01">block_encrypt(Type, Key, Ivec, PlainText) -> CipherText | Error</name> + <name since="OTP R16B01">block_encrypt(AeadType, Key, Ivec, {AAD, PlainText}) -> {CipherText, CipherTag} | Error</name> + <name since="OTP R16B01">block_encrypt(aes_gcm | aes_ccm, Key, Ivec, {AAD, PlainText, TagLength}) -> {CipherText, CipherTag} | Error </name> <fsummary>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher</fsummary> <type> - <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> + <v>Type = <seealso marker="#type-block_cipher_iv">block_cipher_iv()</seealso></v> <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> <v>PlainText = iodata()</v> <v>AAD = IVec = CipherText = CipherTag = binary()</v> <v>TagLength = 1..16</v> + <v>Error = <seealso marker="#type-run_time_error">run_time_error()</seealso></v> </type> <desc> <p>Encrypt <c>PlainText</c> according to <c>Type</c> block cipher. @@ -595,15 +658,17 @@ </func> <func> - <name since="OTP R16B01">block_decrypt(Type, Key, Ivec, CipherText) -> PlainText</name> - <name since="OTP R16B01">block_decrypt(AeadType, Key, Ivec, {AAD, CipherText, CipherTag}) -> PlainText | error</name> + <name since="OTP R16B01">block_decrypt(Type, Key, Ivec, CipherText) -> PlainText | Error</name> + <name since="OTP R16B01">block_decrypt(AeadType, Key, Ivec, {AAD, CipherText, CipherTag}) -> PlainText | Error</name> <fsummary>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher</fsummary> <type> - <v>Type = <seealso marker="#type-block_cipher_with_iv">block_cipher_with_iv()</seealso></v> + <v>Type = <seealso marker="#type-block_cipher_iv">block_cipher_iv()</seealso></v> <v>AeadType = <seealso marker="#type-aead_cipher">aead_cipher()</seealso></v> <v>Key = <seealso marker="#type-key">key()</seealso> | <seealso marker="#type-des3_key">des3_key()</seealso></v> <v>PlainText = iodata()</v> <v>AAD = IVec = CipherText = CipherTag = binary()</v> + <v>Error = BadTag | <seealso marker="#type-run_time_error">run_time_error()</seealso></v> + <v>BadTag = error</v> </type> <desc> <p>Decrypt <c>CipherText</c> according to <c>Type</c> block cipher. @@ -844,6 +909,39 @@ </func> <func> + <name name="hash_info" arity="1" since="OTP 22.0"/> + <fsummary>Information about supported hash algorithms.</fsummary> + <desc> + <p>Provides a map with information about block_size, size and possibly other properties of the + hash algorithm in question. + </p> + <p>For a list of supported hash algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </func> + + <func> + <name name="cipher_info" arity="1" since="OTP 22.0"/> + <fsummary>Information about supported ciphers.</fsummary> + <desc> + <p>Provides a map with information about block_size, key_length, iv_length and possibly other properties of the + cipher algorithm in question. + </p> + <note> + <p>The ciphers <c>aes_cbc</c>, <c>aes_cfb8</c>, <c>aes_cfb128</c>, <c>aes_ctr</c>, + <c>aes_ecb</c>, <c>aes_gcm</c> and <c>aes_ccm</c> + has no keylength in the <c>Type</c> as opposed to for example <c>aes_128_ctr</c>. They adapt to the length of + the key provided in the encrypt and decrypt function. Therefor it is impossible to return a valid keylength + in the map.</p> + <p>Always use a <c>Type</c> with an explicit key length, + </p> + </note> + <p>For a list of supported cipher algorithms, see <seealso marker="#supports-0">supports/0</seealso>. + </p> + </desc> + </func> + + <func> <name name="mod_pow" arity="3" since="OTP R16B01"/> <fsummary>Computes the function: N^P mod M</fsummary> <desc> @@ -1289,8 +1387,8 @@ FloatValue = rand:uniform(). % again <desc> <p> Can be used to determine which crypto algorithms that are supported by the underlying libcrypto library</p> - <p>Note: the <c>rsa_opts</c> entry is in an experimental state and may change or be removed without notice. - No guarantee for the accuarcy of the rsa option's value list should be assumed. + <p>See <seealso marker="#hash_info-1">hash_info/1</seealso> and <seealso marker="#cipher_info-1">cipher_info/1</seealso> + for information about the hash and cipher algorithms. </p> </desc> </func> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 5cf34f8069..fd13481951 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -277,7 +277,13 @@ -type edwards_curve_ed() :: ed25519 | ed448 . %%% --type block_cipher_with_iv() :: cbc_cipher() +-type cipher() :: block_cipher() + | stream_cipher() + | aead_cipher() . + +-type block_cipher() :: block_cipher_iv() | block_cipher_no_iv() . + +-type block_cipher_iv() :: cbc_cipher() | cfb_cipher() | aes_ige256 | blowfish_ofb64 @@ -310,7 +316,7 @@ | des3_cfb . --type block_cipher_without_iv() :: ecb_cipher() . +-type block_cipher_no_iv() :: ecb_cipher() . -type ecb_cipher() :: des_ecb | blowfish_ecb | aes_ecb . -type key() :: iodata(). @@ -330,6 +336,20 @@ -type crypto_integer() :: binary() | integer(). +%%% +%% Exceptions +%% error:badarg +%% error:notsup +-type run_time_error() :: no_return(). + +%% Exceptions +%% error:{badarg,Reason::term()} +%% error:{notsup,Reason::term()} +%% error:{error,Reason::term()} +-type descriptive_error() :: no_return() . + + +%%-------------------------------------------------------------------- -compile(no_native). -on_load(on_load/0). -define(CRYPTO_NIF_VSN,302). @@ -368,10 +388,7 @@ stop() -> | {curves, Curves} | {rsa_opts, RSAopts}, Hashs :: [sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash()], - Ciphers :: [stream_cipher() - | block_cipher_with_iv() | block_cipher_without_iv() - | aead_cipher() - ], + Ciphers :: [cipher()], PKs :: [rsa | dss | ecdsa | dh | ecdh | ec_gf2m], Macs :: [hmac | cmac | poly1305], Curves :: [ec_named_curve() | edwards_curve_dh() | edwards_curve_ed()], @@ -405,14 +422,18 @@ enable_fips_mode(_) -> ?nif_stub. %%% %%%================================================================ --define(HASH_HASH_ALGORITHM, sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() ). - --spec hash_info(Type) -> map() when Type :: ?HASH_HASH_ALGORITHM. +-type hash_algorithm() :: sha1() | sha2() | sha3() | blake2() | ripemd160 | compatibility_only_hash() . +-spec hash_info(Type) -> Result | run_time_error() + when Type :: hash_algorithm(), + Result :: #{size := integer(), + block_size := integer(), + type := integer() + } . hash_info(Type) -> notsup_to_error(hash_info_nif(Type)). --spec hash(Type, Data) -> Digest when Type :: ?HASH_HASH_ALGORITHM, +-spec hash(Type, Data) -> Digest when Type :: hash_algorithm(), Data :: iodata(), Digest :: binary(). hash(Type, Data) -> @@ -422,7 +443,7 @@ hash(Type, Data) -> -opaque hash_state() :: reference(). --spec hash_init(Type) -> State when Type :: ?HASH_HASH_ALGORITHM, +-spec hash_init(Type) -> State when Type :: hash_algorithm(), State :: hash_state(). hash_init(Type) -> notsup_to_error(hash_init_nif(Type)). @@ -448,12 +469,12 @@ hash_final(Context) -> %%%---- HMAC --define(HMAC_HASH_ALGORITHM, sha1() | sha2() | sha3() | compatibility_only_hash()). +-type hmac_hash_algorithm() :: sha1() | sha2() | sha3() | compatibility_only_hash(). %%%---- hmac/3,4 -spec hmac(Type, Key, Data) -> - Mac when Type :: ?HMAC_HASH_ALGORITHM, + Mac when Type :: hmac_hash_algorithm(), Key :: iodata(), Data :: iodata(), Mac :: binary() . @@ -462,7 +483,7 @@ hmac(Type, Key, Data) -> hmac(Type, Key, Data1, undefined, erlang:byte_size(Data1), max_bytes()). -spec hmac(Type, Key, Data, MacLength) -> - Mac when Type :: ?HMAC_HASH_ALGORITHM, + Mac when Type :: hmac_hash_algorithm(), Key :: iodata(), Data :: iodata(), MacLength :: integer(), @@ -477,7 +498,7 @@ hmac(Type, Key, Data, MacLength) -> -opaque hmac_state() :: binary(). -spec hmac_init(Type, Key) -> - State when Type :: ?HMAC_HASH_ALGORITHM, + State when Type :: hmac_hash_algorithm(), Key :: iodata(), State :: hmac_state() . hmac_init(Type, Key) -> @@ -541,79 +562,124 @@ poly1305(Key, Data) -> %%%================================================================ -define(COMPAT(CALL), - try CALL + try begin CALL end catch + error:{error,_} -> + error(badarg); error:{E,_Reason} when E==notsup ; E==badarg -> error(E) end). --spec cipher_info(Type) -> map() when Type :: block_cipher_with_iv() - | aead_cipher() - | block_cipher_without_iv(). +%%%---- Cipher info +%%%---------------------------------------------------------------- +-spec cipher_info(Type) -> Result | run_time_error() + when Type :: cipher(), + Result :: #{key_length := integer(), + iv_length := integer(), + block_size := integer(), + mode := CipherModes, + type := undefined | integer() + }, + CipherModes :: undefined + | cbc_mode + | ccm_mode + | cfb_mode + | ctr_mode + | ecb_mode + | gcm_mode + | ige_mode + | ocb_mode + | ofb_mode + | wrap_mode + | xts_mode + . + +%% These ciphers are not available via the EVP interface on older cryptolibs. +cipher_info(aes_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 32,mode => ctr_mode,type => undefined}; +cipher_info(aes_128_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 16,mode => ctr_mode,type => undefined}; +cipher_info(aes_192_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 24,mode => ctr_mode,type => undefined}; +cipher_info(aes_256_ctr) -> + #{block_size => 1,iv_length => 16,key_length => 32,mode => ctr_mode,type => undefined}; +%% This cipher is handled specialy. +cipher_info(aes_ige256) -> + #{block_size => 16,iv_length => 32,key_length => 16,mode => ige_mode,type => undefined}; cipher_info(Type) -> - cipher_info_nif(Type). + cipher_info_nif(alias(Type)). %%%---- Block ciphers %%%---------------------------------------------------------------- --spec block_encrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> binary(); +-spec block_encrypt(Type::block_cipher_iv(), Key::key()|des3_key(), Ivec::binary(), PlainText::iodata()) -> + binary() | run_time_error(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata()}) -> - {binary(), binary()}; + {binary(), binary()} | run_time_error(); (aes_gcm | aes_ccm, Key::iodata(), Ivec::binary(), {AAD::binary(), PlainText::iodata(), TagLength::1..16}) -> - {binary(), binary()}. + {binary(), binary()} | run_time_error(). -block_encrypt(Type, Key, Ivec, Data) -> - do_block_encrypt(alias(Type), Key, Ivec, Data). - -do_block_encrypt(Type, Key, Ivec, PlainText) when Type =:= aes_ige256 -> +block_encrypt(aes_ige256, Key, Ivec, PlainText) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true)); -do_block_encrypt(Type, Key, Ivec, {AAD, PlainText}) when Type =:= chacha20_poly1305 -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText, 16); - -do_block_encrypt(Type, Key, Ivec, Data) when Type =:= aes_gcm; - Type =:= aes_ccm -> - case Data of - {AAD, PlainText} -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText); - {AAD, PlainText, TagLength} -> - aead_encrypt(Type, Key, Ivec, AAD, PlainText, TagLength) - end; - -do_block_encrypt(Type, Key, Ivec, PlainText) -> - ?COMPAT(crypto_one_shot(Type, Key, Ivec, PlainText, true)). - - --spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary(). - -block_encrypt(Type, Key, PlainText) -> - ?COMPAT(crypto_one_shot(Type, Key, <<>>, PlainText, true)). +block_encrypt(Type, Key0, Ivec, Data) -> + Key = iolist_to_binary(Key0), + ?COMPAT( + case Data of + {AAD, PlainText} -> + aead_encrypt(alias(Type,Key), Key, Ivec, AAD, PlainText, aead_tag_len(Type)); + {AAD, PlainText, TagLength} -> + aead_encrypt(alias(Type,Key), Key, Ivec, AAD, PlainText, TagLength); + PlainText -> + crypto_one_shot(alias(Type,Key), Key, Ivec, PlainText, true) + end). + +-spec block_encrypt(Type::block_cipher_no_iv(), Key::key(), PlainText::iodata()) -> + binary() | run_time_error(). + +block_encrypt(Type, Key0, PlainText) -> + Key = iolist_to_binary(Key0), + ?COMPAT(crypto_one_shot(alias(Type,Key), Key, <<>>, PlainText, true)). + + +aead_tag_len(chacha20_poly1305) -> 16; +aead_tag_len(aes_ccm) -> 12; +aead_tag_len(aes_128_ccm) -> 12; +aead_tag_len(aes_192_ccm) -> 12; +aead_tag_len(aes_256_ccm) -> 12; +aead_tag_len(aes_gcm) -> 16; +aead_tag_len(aes_128_gcm) -> 16; +aead_tag_len(aes_192_gcm) -> 16; +aead_tag_len(aes_256_gcm) -> 16. %%%---------------------------------------------------------------- %%%---------------------------------------------------------------- --spec block_decrypt(Type::block_cipher_with_iv(), Key::key()|des3_key(), Ivec::binary(), Data::iodata()) -> binary(); +-spec block_decrypt(Type::block_cipher_iv(), Key::key()|des3_key(), Ivec::binary(), Data::iodata()) -> + binary() | run_time_error(); (Type::aead_cipher(), Key::iodata(), Ivec::binary(), - {AAD::binary(), Data::iodata(), Tag::binary()}) -> binary() | error. - -block_decrypt(Type, Key, Ivec, Data) -> - do_block_decrypt(alias(Type), Key, Ivec, Data). + {AAD::binary(), Data::iodata(), Tag::binary()}) -> + binary() | error | run_time_error() . -do_block_decrypt(aes_ige256, Key, Ivec, Data) -> +block_decrypt(aes_ige256, Key, Ivec, Data) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); -do_block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; - Type =:= aes_ccm; - Type =:= chacha20_poly1305 -> - aead_decrypt(Type, Key, Ivec, AAD, Data, Tag); +block_decrypt(Type, Key0, Ivec, Data) -> + Key = iolist_to_binary(Key0), + ?COMPAT( + case Data of + {AAD, CryptoText, Tag} -> + aead_decrypt(alias(Type,Key), Key, Ivec, AAD, CryptoText, Tag); + CryptoText -> + crypto_one_shot(alias(Type,Key), Key, Ivec, CryptoText, false) + end). -do_block_decrypt(Type, Key, Ivec, Data) -> - ?COMPAT(crypto_one_shot(Type, Key, Ivec, Data, false)). +-spec block_decrypt(Type::block_cipher_no_iv(), Key::key(), Data::iodata()) -> + binary() | run_time_error(). --spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary(). - -block_decrypt(Type, Key, Data) -> - ?COMPAT(crypto_one_shot(Type, Key, <<>>, Data, false)). +block_decrypt(Type, Key0, CryptoText) -> + Key = iolist_to_binary(Key0), + ?COMPAT(crypto_one_shot(alias(Type,Key), Key, <<>>, CryptoText, false)). %%%-------- Stream ciphers API @@ -630,32 +696,34 @@ block_decrypt(Type, Key, Data) -> | chacha20 . %%%---- stream_init --spec stream_init(Type, Key, IVec) -> State | no_return() +-spec stream_init(Type, Key, IVec) -> State | run_time_error() when Type :: stream_cipher_iv(), Key :: iodata(), IVec ::binary(), State :: stream_state() . -stream_init(Type, Key, IVec) when is_binary(IVec) -> - Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), - iolist_to_binary(Key), iolist_to_binary(IVec), +stream_init(Type, Key0, IVec) when is_binary(IVec) -> + Key = iolist_to_binary(Key0), + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type,Key), + Key, iolist_to_binary(IVec), undefined) ), {Type, {Ref,flg_undefined}}. --spec stream_init(Type, Key) -> State | no_return() +-spec stream_init(Type, Key) -> State | run_time_error() when Type :: stream_cipher_no_iv(), Key :: iodata(), State :: stream_state() . -stream_init(rc4 = Type, Key) -> - Ref = ?COMPAT(ng_crypto_init_nif(alias(Type), - iolist_to_binary(Key), <<>>, +stream_init(rc4 = Type, Key0) -> + Key = iolist_to_binary(Key0), + Ref = ?COMPAT(ng_crypto_init_nif(alias(Type,Key), + Key, <<>>, undefined) ), {Type, {Ref,flg_undefined}}. %%%---- stream_encrypt --spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | no_return() +-spec stream_encrypt(State, PlainText) -> {NewState, CipherText} | run_time_error() when State :: stream_state(), PlainText :: iodata(), NewState :: stream_state(), @@ -664,7 +732,7 @@ stream_encrypt(State, Data) -> crypto_stream_emulate(State, Data, true). %%%---- stream_decrypt --spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | no_return() +-spec stream_decrypt(State, CipherText) -> {NewState, PlainText} | run_time_error() when State :: stream_state(), CipherText :: iodata(), NewState :: stream_state(), @@ -723,8 +791,8 @@ next_iv(Type, Data, _Ivec) -> %%% Create and initialize a new state for encryption or decryption %%% --spec crypto_init(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() - when Cipher :: block_cipher_without_iv() +-spec crypto_init(Cipher, Key, EncryptFlag) -> State | descriptive_error() + when Cipher :: block_cipher_no_iv() | stream_cipher_no_iv(), Key :: iodata(), EncryptFlag :: boolean(), @@ -734,9 +802,9 @@ crypto_init(Cipher, Key, EncryptFlag) -> ng_crypto_init_nif(alias(Cipher), iolist_to_binary(Key), <<>>, EncryptFlag). --spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | ng_crypto_error() +-spec crypto_init(Cipher, Key, IV, EncryptFlag) -> State | descriptive_error() when Cipher :: stream_cipher_iv() - | block_cipher_with_iv(), + | block_cipher_iv(), Key :: iodata(), IV :: iodata(), EncryptFlag :: boolean(), @@ -747,9 +815,9 @@ crypto_init(Cipher, Key, IV, EncryptFlag) -> %%%---------------------------------------------------------------- --spec crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> State | ng_crypto_error() +-spec crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> State | descriptive_error() when Cipher :: stream_cipher_iv() - | block_cipher_with_iv(), + | block_cipher_iv(), Key :: iodata(), EncryptFlag :: boolean(), State :: crypto_state() . @@ -764,7 +832,7 @@ crypto_init_dyn_iv(Cipher, Key, EncryptFlag) -> %%% blocksize. %%% --spec crypto_update(State, Data) -> Result | ng_crypto_error() +-spec crypto_update(State, Data) -> Result | descriptive_error() when State :: crypto_state(), Data :: iodata(), Result :: binary() . @@ -778,7 +846,7 @@ crypto_update(State, Data0) -> %%%---------------------------------------------------------------- --spec crypto_update_dyn_iv(State, Data, IV) -> Result | ng_crypto_error() +-spec crypto_update_dyn_iv(State, Data, IV) -> Result | descriptive_error() when State :: crypto_state(), Data :: iodata(), IV :: iodata(), @@ -798,15 +866,16 @@ crypto_update_dyn_iv(State, Data0, IV) -> %%% The size must be an integer multiple of the crypto's blocksize. %%% --spec crypto_one_shot(Cipher, Key, IV, Data, EncryptFlag) -> Result | ng_crypto_error() - when Cipher :: stream_cipher() - | block_cipher_with_iv() - | block_cipher_without_iv(), - Key :: iodata(), - IV :: iodata() | undefined, - Data :: iodata(), - EncryptFlag :: boolean(), - Result :: binary() . +-spec crypto_one_shot(Cipher, Key, IV, Data, EncryptFlag) -> + Result | descriptive_error() + when Cipher :: stream_cipher() + | block_cipher(), + Key :: iodata(), + IV :: iodata() | undefined, + Data :: iodata(), + EncryptFlag :: boolean(), + Result :: binary() . + crypto_one_shot(Cipher, Key, undefined, Data, EncryptFlag) -> crypto_one_shot(Cipher, Key, <<>>, Data, EncryptFlag); @@ -823,21 +892,25 @@ crypto_one_shot(Cipher, Key, IV, Data0, EncryptFlag) -> %%%---------------------------------------------------------------- %%% NIFs --type ng_crypto_error() :: no_return() . +-spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> + crypto_state() | descriptive_error() + ; (crypto_state(), <<>>, <<>>, boolean()) + -> crypto_state() | descriptive_error(). --spec ng_crypto_init_nif(atom(), binary(), binary()|undefined, boolean()|undefined ) -> crypto_state() | ng_crypto_error() - ; (crypto_state(), <<>>, <<>>, boolean()) -> crypto_state() | ng_crypto_error(). ng_crypto_init_nif(_Cipher, _Key, _IVec, _EncryptFlg) -> ?nif_stub. --spec ng_crypto_update_nif(crypto_state(), binary()) -> binary() | ng_crypto_error() . +-spec ng_crypto_update_nif(crypto_state(), binary()) -> + binary() | descriptive_error() . ng_crypto_update_nif(_State, _Data) -> ?nif_stub. --spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> binary() | ng_crypto_error() . +-spec ng_crypto_update_nif(crypto_state(), binary(), binary()) -> + binary() | descriptive_error() . ng_crypto_update_nif(_State, _Data, _IV) -> ?nif_stub. --spec ng_crypto_one_shot_nif(atom(), binary(), binary(), binary(), boolean() ) -> binary() | ng_crypto_error(). +-spec ng_crypto_one_shot_nif(atom(), binary(), binary(), binary(), boolean() ) -> + binary() | descriptive_error(). ng_crypto_one_shot_nif(_Cipher, _Key, _IVec, _Data, _EncryptFlg) -> ?nif_stub. %%%---------------------------------------------------------------- @@ -859,6 +932,44 @@ alias(aes_cbc256) -> aes_256_cbc; alias(Alg) -> Alg. + +%%%---- des_ede3_cbc +alias(des3_cbc, _) -> des_ede3_cbc; +alias(des_ede3, _) -> des_ede3_cbc; +%%%---- des_ede3_cfb +alias(des_ede3_cbf,_ ) -> des_ede3_cfb; +alias(des3_cbf, _) -> des_ede3_cfb; +alias(des3_cfb, _) -> des_ede3_cfb; +%%%---- aes_*_cbc +alias(aes_cbc128, _) -> aes_128_cbc; +alias(aes_cbc256, _) -> aes_256_cbc; + +alias(aes_cbc, Key) when size(Key)==128 -> aes_128_cbc; +alias(aes_cbc, Key) when size(Key)==192 -> aes_192_cbc; +alias(aes_cbc, Key) when size(Key)==256 -> aes_256_cbc; + +alias(aes_cfb8, Key) when size(Key)==128 -> aes_128_cfb8; +alias(aes_cfb8, Key) when size(Key)==192 -> aes_192_cfb8; +alias(aes_cfb8, Key) when size(Key)==256 -> aes_256_cfb8; + +alias(aes_cfb128, Key) when size(Key)==128 -> aes_128_cfb128; +alias(aes_cfb128, Key) when size(Key)==192 -> aes_192_cfb128; +alias(aes_cfb128, Key) when size(Key)==256 -> aes_256_cfb128; + +alias(aes_ctr, Key) when size(Key)==128 -> aes_128_ctr; +alias(aes_ctr, Key) when size(Key)==192 -> aes_192_ctr; +alias(aes_ctr, Key) when size(Key)==256 -> aes_256_ctr; + +alias(aes_gcm, Key) when size(Key)==128 -> aes_128_gcm; +alias(aes_gcm, Key) when size(Key)==192 -> aes_192_gcm; +alias(aes_gcm, Key) when size(Key)==256 -> aes_256_gcm; + +alias(aes_ccm, Key) when size(Key)==128 -> aes_128_ccm; +alias(aes_ccm, Key) when size(Key)==192 -> aes_192_ccm; +alias(aes_ccm, Key) when size(Key)==256 -> aes_256_ccm; + +alias(Alg, _) -> Alg. + %%%================================================================ %%% %%% RAND - pseudo random numbers using RN_ and BN_ functions in crypto lib @@ -1949,9 +2060,6 @@ cipher_info_nif(_Type) -> ?nif_stub. %% AES - in Galois/Counter Mode (GCM) %% %% The default tag length is EVP_GCM_TLS_TAG_LEN(16), -aead_encrypt(Type=aes_ccm, Key, Ivec, AAD, In) -> aead_encrypt(Type, Key, Ivec, AAD, In, 12); -aead_encrypt(Type=aes_gcm, Key, Ivec, AAD, In) -> aead_encrypt(Type, Key, Ivec, AAD, In, 16). - aead_encrypt(_Type, _Key, _Ivec, _AAD, _In, _TagLength) -> ?nif_stub. aead_decrypt(_Type, _Key, _Ivec, _AAD, _In, _Tag) -> ?nif_stub. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 7dbbde68e9..ce5097de47 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -323,12 +323,11 @@ end_per_group(_GroupName, Config) -> init_per_testcase(info, Config) -> Config; init_per_testcase(cmac, Config) -> - case crypto:info_lib() of - [{<<"OpenSSL">>,LibVer,_}] when is_integer(LibVer), LibVer > 16#10001000 -> + case is_supported(cmac) of + true -> Config; - _Else -> - % The CMAC functionality was introduced in OpenSSL 1.0.1 - {skip, "OpenSSL is too old"} + false -> + {skip, "CMAC is not supported"} end; init_per_testcase(generate, Config) -> case proplists:get_value(type, Config) of @@ -848,7 +847,8 @@ cipher_info(Config) when is_list(Config) -> #{type := _,key_length := _,iv_length := _, block_size := _,mode := _} = crypto:cipher_info(aes_128_cbc), {'EXIT',_} = (catch crypto:cipher_info(not_a_cipher)), - ok. + lists:foreach(fun(C) -> crypto:cipher_info(C) end, + proplists:get_value(ciphers, crypto:supports())). %%-------------------------------------------------------------------- hash_info() -> @@ -856,7 +856,8 @@ hash_info() -> hash_info(Config) when is_list(Config) -> #{type := _,size := _,block_size := _} = crypto:hash_info(sha256), {'EXIT',_} = (catch crypto:hash_info(not_a_hash)), - ok. + lists:foreach(fun(H) -> crypto:hash_info(H) end, + proplists:get_value(hashs, crypto:supports())). %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ diff --git a/lib/dialyzer/src/Makefile b/lib/dialyzer/src/Makefile index fc08e7ca2f..bddd761705 100644 --- a/lib/dialyzer/src/Makefile +++ b/lib/dialyzer/src/Makefile @@ -90,8 +90,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += +warn_export_vars +warn_unused_import +warn_untyped_record +warn_missing_spec +warnings_as_errors +ERL_COMPILE_FLAGS += +warn_export_vars +warn_unused_import +warn_untyped_record +warn_missing_spec # ---------------------------------------------------- # Targets diff --git a/lib/hipe/cerl/Makefile b/lib/hipe/cerl/Makefile index f653dce36f..5c367b5b77 100644 --- a/lib/hipe/cerl/Makefile +++ b/lib/hipe/cerl/Makefile @@ -66,7 +66,10 @@ DOC_FILES= $(MODULES:%=$(DOCS)/%.html) include ../native.mk -ERL_COMPILE_FLAGS += +inline -Werror +warn_export_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record +ERL_COMPILE_FLAGS += +inline +warn_export_vars +warn_unused_import +warn_missing_spec #+warn_untyped_record +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # ---------------------------------------------------- # Targets diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index ffe81ef9b8..8e7e56b6c4 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -1189,6 +1189,21 @@ trans_fun([raw_raise|Instructions], Env) -> [hipe_icode:mk_primop(Dst,raw_raise,Vars) | trans_fun(Instructions, Env)]; %%-------------------------------------------------------------------- +%% New binary matching added in OTP 22. +%%-------------------------------------------------------------------- +%%--- bs_get_tail --- +trans_fun([{bs_get_tail=Name,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_start_match3 --- +trans_fun([{bs_start_match3=Name,_,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_get_position --- +trans_fun([{bs_get_position=Name,_,_,_}|_Instructions], _Env) -> + nyi(Name); +%%--- bs_set_position --- +trans_fun([{bs_set_position=Name,_,_}|_Instructions], _Env) -> + nyi(Name); +%%-------------------------------------------------------------------- %%--- ERROR HANDLING --- %%-------------------------------------------------------------------- trans_fun([X|_], _) -> @@ -1196,6 +1211,9 @@ trans_fun([X|_], _) -> trans_fun([], _) -> []. +nyi(Name) -> + throw({unimplemented_instruction,Name}). + %%-------------------------------------------------------------------- %% trans_call and trans_enter generate correct Icode calls/tail-calls, %% recognizing explicit fails. diff --git a/lib/hipe/llvm/Makefile b/lib/hipe/llvm/Makefile index 817ff67dcd..9f7a2def6d 100644 --- a/lib/hipe/llvm/Makefile +++ b/lib/hipe/llvm/Makefile @@ -70,7 +70,10 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) include ../native.mk -ERL_COMPILE_FLAGS += -Werror +inline +warn_export_vars #+warn_missing_spec +ERL_COMPILE_FLAGS += +inline +warn_export_vars #+warn_missing_spec +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # if in 32 bit backend define BIT32 symbol ifneq ($(BITS64),yes) diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 2348e9b1f6..094b7bc508 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -583,9 +583,8 @@ fix_beam_exports([], Exports) -> Exports. get_beam_icode(Mod, {BeamCode, Exports}, File, Options) -> - {ok, Icode} = - ?option_time((catch {ok, hipe_beam_to_icode:module(BeamCode, Options)}), - "BEAM-to-Icode", Options), + Icode = ?option_time(hipe_beam_to_icode:module(BeamCode, Options), + "BEAM-to-Icode", Options), BeamBin = get_beam_code(File), {{Mod, Exports, Icode}, BeamBin}. @@ -662,9 +661,12 @@ run_compiler_1(Name, DisasmFun, IcodeFun, Options) -> {Icode, WholeModule} = IcodeFun(Code, Opts), CompRes = compile_finish(Icode, WholeModule, Opts), compiler_return(CompRes, Parent) - catch error:Error:StackTrace -> + catch + error:Error:StackTrace -> print_crash_message(Name, Error, StackTrace), - exit(Error) + exit(Error); + throw:{unimplemented_instruction,_Instruction}=Error -> + exit(Error) end end), Timeout = case proplists:get_value(timeout, Options) of diff --git a/lib/hipe/rtl/Makefile b/lib/hipe/rtl/Makefile index becdd0b7d8..0c0f6e24f5 100644 --- a/lib/hipe/rtl/Makefile +++ b/lib/hipe/rtl/Makefile @@ -75,7 +75,10 @@ TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) include ../native.mk -ERL_COMPILE_FLAGS += -Werror +inline +warn_unused_import +warn_export_vars +ERL_COMPILE_FLAGS += +inline +warn_unused_import +warn_export_vars +ifneq ($(NATIVE_LIBS_ENABLED),yes) +ERL_COMPILE_FLAGS += -Werror +endif # ---------------------------------------------------- # Targets diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 8357e02014..d4b33ae2c6 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -339,14 +339,6 @@ end_per_testcase(Case, Config) end_per_testcase(_Case, _Config) -> ok. -is_ipv6_supported() -> - case gen_udp:open(0, [inet6]) of - {ok, Socket} -> - gen_udp:close(Socket), - true; - _ -> - false - end. %%-------------------------------------------------------------------- @@ -1612,7 +1604,8 @@ post_with_content_type(Config) when is_list(Config) -> %%-------------------------------------------------------------------- request_options() -> - [{doc, "Test http get request with socket options against local server (IPv6)"}]. + [{require, ipv6_hosts}, + {doc, "Test http get request with socket options against local server (IPv6)"}]. request_options(Config) when is_list(Config) -> Request = {url(group_name(Config), "/dummy.html", Config), []}, {ok, {{_,200,_}, [_ | _], _ = [_ | _]}} = httpc:request(get, Request, [], @@ -2945,3 +2938,12 @@ receive_stream_n(Ref, N) -> ct:pal("Data: ~p", [Data]), receive_stream_n(Ref, N-1) end. + +is_ipv6_supported() -> + {ok, Hostname0} = inet:gethostname(), + try + lists:member(list_to_atom(Hostname0), ct:get_config(ipv6_hosts)) + catch + _: _ -> + false + end. diff --git a/lib/kernel/doc/src/logger_disk_log_h.xml b/lib/kernel/doc/src/logger_disk_log_h.xml index 5b2374690e..aa577f3c62 100644 --- a/lib/kernel/doc/src/logger_disk_log_h.xml +++ b/lib/kernel/doc/src/logger_disk_log_h.xml @@ -66,7 +66,7 @@ corresponds to the <c>name</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to the same name as the handler identity, in the current directory.</p> @@ -77,7 +77,7 @@ corresponds to the <c>type</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>wrap</c>.</p> </item> @@ -88,7 +88,7 @@ corresponds to the <c>MaxNoFiles</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>10</c>.</p> <p>The setting has no effect on a halt log.</p> @@ -101,7 +101,7 @@ corresponds to the <c>MaxNoBytes</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>1048576</c> bytes for a wrap log, and <c>infinity</c> for a halt log.</p> diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index 5ed1a2f210..5ac4f58d12 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -58,7 +58,7 @@ <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> <p>Specifies the log destination.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>standard_io</c>, unless parameter <seealso marker="#file"><c>file</c></seealso> is @@ -68,7 +68,7 @@ <item> <p>This specifies the name of the log file when the handler is of type <c>file</c>.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to the same name as the handler identity, in the current directory.</p> @@ -93,9 +93,9 @@ or <c>{delayed_write,Size,Delay}</c> is found in the list, <c>delayed_write</c> is added.</item> </list> - <p>Log files are always UTF-8 encoded. The encoding can not be + <p>Log files are always UTF-8 encoded. The encoding cannot be changed by setting the mode <c>{encoding,Encoding}</c>.</p> - <p>The value is set when the handler is added, and it can not + <p>The value is set when the handler is added, and it cannot be changed in runtime.</p> <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> </item> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 0c187eb19f..61bd598145 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a performance regression when reading files + opened with the <c>compressed</c> flag.</p> + <p> + Own Id: OTP-15706 Aux Id: ERIERL-336 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/examples/gen_tcp_dist/Makefile b/lib/kernel/examples/gen_tcp_dist/Makefile index 65513a1729..0c916835ea 100644 --- a/lib/kernel/examples/gen_tcp_dist/Makefile +++ b/lib/kernel/examples/gen_tcp_dist/Makefile @@ -2,9 +2,7 @@ RM=rm -f CP=cp EBIN=ebin ERLC=erlc -# Works if building in open source source tree -KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include -ERLCFLAGS+= -W -I$(KERNEL_INCLUDE) +ERLCFLAGS+= -W MODULES=gen_tcp_dist diff --git a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl index 98554ed805..cda4c470f9 100644 --- a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl +++ b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl @@ -53,10 +53,10 @@ -import(error_logger,[error_msg/2]). --include("net_address.hrl"). +-include_lib("kernel/include/net_address.hrl"). --include("dist.hrl"). --include("dist_util.hrl"). +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). %% ------------------------------------------------------------ %% Select this protocol based on node name @@ -679,7 +679,14 @@ dist_cntrlr_setup_loop(Socket, TickHandler, Sup) -> %% From now on we execute on normal priority process_flag(priority, normal), erlang:dist_ctrl_get_data_notification(DHandle), - dist_cntrlr_output_loop(DHandle, Socket) + case init:get_argument(gen_tcp_dist_output_loop) of + error -> + dist_cntrlr_output_loop(DHandle, Socket); + {ok, [[ModStr, FuncStr]]} -> % For testing... + apply(list_to_atom(ModStr), + list_to_atom(FuncStr), + [DHandle, Socket]) + end end. %% We use active 10 for good throughput while still diff --git a/lib/kernel/src/Makefile b/lib/kernel/src/Makefile index 43b776f37e..fcb599859b 100644 --- a/lib/kernel/src/Makefile +++ b/lib/kernel/src/Makefile @@ -175,8 +175,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -Werror +ERL_COMPILE_FLAGS += -I../include # ---------------------------------------------------- # Targets diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 9a8091fb2e..7715dca7c6 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -537,14 +537,12 @@ check_conf_data(ConfData) when is_list(ConfData) -> {AppName, List} when is_atom(AppName), is_list(List) -> case lists:keymember(AppName, 1, ConfDataRem) of true -> - ?LOG_WARNING("duplicate application config: " ++ atom_to_list(AppName)); + {error, "duplicate application config: " ++ atom_to_list(AppName)}; false -> - ok - end, - - case check_para(List, AppName) of - ok -> check_conf_data(ConfDataRem); - Error -> Error + case check_para(List, AppName) of + ok -> check_conf_data(ConfDataRem); + Error -> Error + end end; {AppName, List} when is_list(List) -> ErrMsg = "application: " @@ -570,15 +568,14 @@ check_para([], _AppName) -> check_para([{Para, Val} | ParaList], AppName) when is_atom(Para) -> case lists:keymember(Para, 1, ParaList) of true -> - ?LOG_WARNING("application: " ++ atom_to_list(AppName) ++ - "; duplicate parameter: " ++ atom_to_list(Para)); + ErrMsg = "application: " ++ atom_to_list(AppName) + ++ "; duplicate parameter: " ++ atom_to_list(Para), + {error, ErrMsg}; false -> - ok - end, - - case check_para_value(Para, Val, AppName) of - ok -> check_para(ParaList, AppName); - {error, _} = Error -> Error + case check_para_value(Para, Val, AppName) of + ok -> check_para(ParaList, AppName); + {error, _} = Error -> Error + end end; check_para([{Para, _Val} | _ParaList], AppName) -> {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter name: " ++ diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index a38522eb5c..3875074d74 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,10 +50,9 @@ %% This is for backward compatibility only; the functionality is broken. -define(WARN_DUPLICATED_NAME, global_multi_name_action). -%% Undocumented Kernel variable. Set this to 0 (zero) to get the old -%% behaviour. +%% Undocumented Kernel variable. -define(N_CONNECT_RETRIES, global_connect_retries). --define(DEFAULT_N_CONNECT_RETRIES, 5). +-define(DEFAULT_N_CONNECT_RETRIES, 0). %%% In certain places in the server, calling io:format hangs everything, %%% so we'd better use erlang:display/1. @@ -125,16 +124,12 @@ %%% There are also ETS tables used for bookkeeping of locks and names %%% (the first position is the key): %%% -%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,RPid,ref()]} +%%% global_locks (set): {ResourceId, LockRequesterId, [{Pid,ref()]} %%% Pid is locking ResourceId, ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). -%%% global_names (set): {Name, Pid, Method, RPid, ref()} +%%% global_names (set): {Name, Pid, Method, ref()} %%% Registered names. ref() is the monitor ref. -%%% RPid =/= Pid if there is an extra process calling erlang:monitor(). %%% global_names_ext (set): {Name, Pid, RegNode} %%% External registered names (C-nodes). -%%% (The RPid:s can be removed when/if erlang:monitor() returns before -%%% trying to connect to the other node.) %%% %%% Helper tables: %%% global_pid_names (bag): {Pid, Name} | {ref(), Name} @@ -310,7 +305,7 @@ re_register_name(Name, Pid, Method0) when is_pid(Pid) -> -spec registered_names() -> [Name] when Name :: term(). registered_names() -> - MS = ets:fun2ms(fun({Name,_Pid,_M,_RP,_R}) -> Name end), + MS = ets:fun2ms(fun({Name,_Pid,_M,_R}) -> Name end), ets:select(global_names, MS). %%----------------------------------------------------------------- @@ -1235,7 +1230,7 @@ ins_name_ext(Name, Pid, Method, RegNode, FromPidOrNode, ExtraInfo, S0) -> where(Name) -> case ets:lookup(global_names, Name) of - [{_Name, Pid, _Method, _RPid, _Ref}] -> + [{_Name, Pid, _Method, _Ref}] -> if node(Pid) == node() -> case is_process_alive(Pid) of true -> Pid; @@ -1272,10 +1267,10 @@ can_set_lock({ResourceId, LockRequesterId}) -> end. insert_lock({ResourceId, LockRequesterId}=Id, Pid, PidRefs, S) -> - {RPid, Ref} = do_monitor(Pid), + Ref = erlang:monitor(process, Pid), true = ets:insert(global_pid_ids, {Pid, ResourceId}), true = ets:insert(global_pid_ids, {Ref, ResourceId}), - Lock = {ResourceId, LockRequesterId, [{Pid,RPid,Ref} | PidRefs]}, + Lock = {ResourceId, LockRequesterId, [{Pid,Ref} | PidRefs]}, true = ets:insert(global_locks, Lock), trace_message(S, {ins_lock, node(Pid)}, [Id, Pid]). @@ -1293,10 +1288,9 @@ handle_del_lock({ResourceId, LockReqId}, Pid, S0) -> _ -> S0 end. -remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> +remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,Ref}], Down, S0) -> ?trace({remove_lock_1, {id,ResourceId},{pid,Pid}}), true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete(global_locks, ResourceId), true = ets:delete_object(global_pid_ids, {Pid, ResourceId}), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), @@ -1309,9 +1303,8 @@ remove_lock(ResourceId, LockRequesterId, Pid, [{Pid,RPid,Ref}], Down, S0) -> remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> ?trace({remove_lock_2, {id,ResourceId},{pid,Pid}}), PidRefs = case lists:keyfind(Pid, 1, PidRefs0) of - {Pid, RPid, Ref} -> + {Pid, Ref} -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), true = ets:delete_object(global_pid_ids, {Ref, ResourceId}), lists:keydelete(Pid, 1, PidRefs0); @@ -1324,11 +1317,6 @@ remove_lock(ResourceId, LockRequesterId, Pid, PidRefs0, _Down, S) -> trace_message(S, {rem_lock, node(Pid)}, [{ResourceId, LockRequesterId}, Pid]). -kill_monitor_proc(Pid, Pid) -> - ok; -kill_monitor_proc(RPid, _Pid) -> - exit(RPid, kill). - do_ops(Ops, ConnNode, Names_ext, ExtraInfo, S0) -> ?trace({do_ops, {ops,Ops}}), @@ -1394,8 +1382,8 @@ sync_other(Node, N) -> % exit(normal). insert_global_name(Name, Pid, Method, FromPidOrNode, ExtraInfo, S) -> - {RPid, Ref} = do_monitor(Pid), - true = ets:insert(global_names, {Name, Pid, Method, RPid, Ref}), + Ref = erlang:monitor(process, Pid), + true = ets:insert(global_names, {Name, Pid, Method, Ref}), true = ets:insert(global_pid_names, {Pid, Name}), true = ets:insert(global_pid_names, {Ref, Name}), case lock_still_set(FromPidOrNode, ExtraInfo, S) of @@ -1437,7 +1425,7 @@ extra_info(Tag, ExtraInfo) -> del_name(Ref, S) -> NameL = [Name || {_, Name} <- ets:lookup(global_pid_names, Ref), - {_, _Pid, _Method, _RPid, Ref1} <- + {_, _Pid, _Method, Ref1} <- ets:lookup(global_names, Name), Ref1 =:= Ref], case NameL of @@ -1450,24 +1438,23 @@ del_name(Ref, S) -> %% Keeps the entry in global_names for whereis_name/1. delete_global_name_keep_pid(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> - delete_global_name2(Name, Pid, RPid, Ref, S); + [{Name, Pid, _Method, Ref}] -> + delete_global_name2(Name, Pid, Ref, S); [] -> S end. delete_global_name2(Name, S) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, RPid, Ref}] -> + [{Name, Pid, _Method, Ref}] -> true = ets:delete(global_names, Name), - delete_global_name2(Name, Pid, RPid, Ref, S); + delete_global_name2(Name, Pid, Ref, S); [] -> S end. -delete_global_name2(Name, Pid, RPid, Ref, S) -> +delete_global_name2(Name, Pid, Ref, S) -> true = erlang:demonitor(Ref, [flush]), - kill_monitor_proc(RPid, Pid), delete_global_name(Name, Pid), ?trace({delete_global_name,{item,Name},{pid,Pid}}), true = ets:delete_object(global_pid_names, {Pid, Name}), @@ -1929,9 +1916,9 @@ reset_node_state(Node) -> %% from the same partition. exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> case ets:lookup(global_names, Name) of - [{Name, Pid, _Method, _RPid2, _Ref2}] -> + [{Name, Pid, _Method, _Ref2}] -> exchange_names(Tail, Node, Ops, Res); - [{Name, Pid2, Method2, _RPid2, _Ref2}] when node() < Node -> + [{Name, Pid2, Method2, _Ref2}] when node() < Node -> %% Name clash! Add the result of resolving to Res(olved). %% We know that node(Pid) =/= node(), so we don't %% need to link/unlink to Pid. @@ -1960,7 +1947,7 @@ exchange_names([{Name, Pid, Method} | Tail], Node, Ops, Res) -> Op = {delete, Name}, exchange_names(Tail, Node, [Op | Ops], [Op | Res]) end; - [{Name, _Pid2, _Method, _RPid, _Ref}] -> + [{Name, _Pid2, _Method, _Ref}] -> %% The other node will solve the conflict. exchange_names(Tail, Node, Ops, Res); _ -> @@ -2036,7 +2023,7 @@ pid_is_locking(Pid, PidRefs) -> delete_lock(Ref, S0) -> Locks = pid_locks(Ref), F = fun({ResourceId, LockRequesterId, PidRefs}, S) -> - {Pid, _RPid, Ref} = lists:keyfind(Ref, 3, PidRefs), + {Pid, Ref} = lists:keyfind(Ref, 2, PidRefs), remove_lock(ResourceId, LockRequesterId, Pid, PidRefs, true, S) end, lists:foldl(F, S0, Locks). @@ -2046,10 +2033,10 @@ pid_locks(Ref) -> ets:lookup(global_locks, ResourceId) end, ets:lookup(global_pid_ids, Ref)), [Lock || Lock = {_Id, _Req, PidRefs} <- L, - rpid_is_locking(Ref, PidRefs)]. + ref_is_locking(Ref, PidRefs)]. -rpid_is_locking(Ref, PidRefs) -> - lists:keyfind(Ref, 3, PidRefs) =/= false. +ref_is_locking(Ref, PidRefs) -> + lists:keyfind(Ref, 2, PidRefs) =/= false. handle_nodedown(Node, S) -> %% DOWN signals from monitors have removed locks and registered names. @@ -2062,7 +2049,7 @@ handle_nodedown(Node, S) -> get_names() -> ets:select(global_names, - ets:fun2ms(fun({Name, Pid, Method, _RPid, _Ref}) -> + ets:fun2ms(fun({Name, Pid, Method, _Ref}) -> {Name, Pid, Method} end)). @@ -2205,24 +2192,6 @@ unexpected_message(Message, What) -> %%% Utilities -%% When/if erlang:monitor() returns before trying to connect to the -%% other node this function can be removed. -do_monitor(Pid) -> - case (node(Pid) =:= node()) orelse lists:member(node(Pid), nodes()) of - true -> - %% Assume the node is still up - {Pid, erlang:monitor(process, Pid)}; - false -> - F = fun() -> - Ref = erlang:monitor(process, Pid), - receive - {'DOWN', Ref, process, Pid, _Info} -> - exit(normal) - end - end, - erlang:spawn_monitor(F) - end. - intersection(_, []) -> []; intersection(L1, L2) -> diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 8fa3f5c588..aca3247c8f 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -43,7 +43,9 @@ {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.2$">>,[restart_new_emulator]}, {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.3$">>,[restart_new_emulator]}, + {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -60,4 +62,6 @@ {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.2$">>,[restart_new_emulator]}, {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.3$">>,[restart_new_emulator]}, + {<<"^6\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/raw_file_io_inflate.erl b/lib/kernel/src/raw_file_io_inflate.erl index 7e9780310c..d3ed02dd03 100644 --- a/lib/kernel/src/raw_file_io_inflate.erl +++ b/lib/kernel/src/raw_file_io_inflate.erl @@ -26,7 +26,7 @@ -include("file_int.hrl"). --define(INFLATE_CHUNK_SIZE, (1 bsl 10)). +-define(INFLATE_CHUNK_SIZE, (8 bsl 10)). -define(GZIP_WBITS, (16 + 15)). callback_mode() -> state_functions. diff --git a/lib/kernel/test/Makefile b/lib/kernel/test/Makefile index d203597fc2..6763a04d9f 100644 --- a/lib/kernel/test/Makefile +++ b/lib/kernel/test/Makefile @@ -57,6 +57,7 @@ MODULES= \ prim_file_SUITE \ ram_file_SUITE \ gen_tcp_api_SUITE \ + gen_tcp_dist \ gen_tcp_echo_SUITE \ gen_tcp_misc_SUITE \ gen_udp_SUITE \ @@ -137,7 +138,10 @@ TARGETS = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) # Targets # ---------------------------------------------------- -make_emakefile: +gen_tcp_dist.erl: ../examples/gen_tcp_dist/src/gen_tcp_dist.erl + cp $< $@ + +make_emakefile: $(ERL_FILES) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' \ > $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) \ diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 94d7c17712..1ab554db7c 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -2020,18 +2020,11 @@ set_env_errors(Conf) when is_list(Conf) -> "application: kernel; erroneous parameter: distributed" = badarg_msg(fun() -> application:set_env([{kernel, [{distributed, config}]}]) end), - %% This will raise in the future - ct:capture_start(), - _ = application:set_env([{foo, []}, {foo, []}]), - timer:sleep(100), - ct:capture_stop(), - [_ | _] = string:find(ct:capture_get(), "duplicate application config: foo"), - - ct:capture_start(), - _ = application:set_env([{foo, [{bar, baz}, {bar, bat}]}]), - timer:sleep(100), - ct:capture_stop(), - [_ | _] = string:find(ct:capture_get(), "application: foo; duplicate parameter: bar"), + "duplicate application config: foo" = + badarg_msg(fun() -> application:set_env([{foo, []}, {foo, []}]) end), + + "application: foo; duplicate parameter: bar" = + badarg_msg(fun() -> application:set_env([{foo, [{bar, baz}, {bar, bat}]}]) end), ok. diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 64e0b9d8dd..4f0847084f 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -25,8 +25,8 @@ -export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2]). -export([set_path/1, get_path/1, add_path/1, add_paths/1, del_path/1, replace_path/1, load_file/1, load_abs/1, ensure_loaded/1, - delete/1, purge/1, purge_many_exits/1, soft_purge/1, is_loaded/1, - all_loaded/1, + delete/1, purge/1, purge_many_exits/0, purge_many_exits/1, + soft_purge/1, is_loaded/1, all_loaded/1, load_binary/1, dir_req/1, object_code/1, set_path_file/1, upgrade/1, sticky_dir/1, pa_pz_option/1, add_del_path/1, @@ -55,7 +55,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,5}}]. + {timetrap,{seconds,30}}]. all() -> [set_path, get_path, add_path, add_paths, del_path, @@ -140,6 +140,11 @@ end_per_testcase(on_load_embedded, Config) -> LinkName = proplists:get_value(link_name, Config), _ = del_link(LinkName), end_per_testcase(Config); +end_per_testcase(upgrade, Config) -> + %% Make sure tracing is turned off even if the test times out. + erlang:trace_pattern({error_handler,undefined_function,3}, false, [global]), + erlang:trace(self(), false, [call]), + end_per_testcase(Config); end_per_testcase(_Func, Config) -> end_per_testcase(Config). @@ -391,6 +396,9 @@ purge(Config) when is_list(Config) -> process_flag(trap_exit, OldFlag), ok. +purge_many_exits() -> + [{timetrap, {minutes, 2}}]. + purge_many_exits(Config) when is_list(Config) -> OldFlag = process_flag(trap_exit, true), @@ -1556,6 +1564,11 @@ on_load_update_code_1(3, Mod) -> %% Test -on_load while trace feature 'on_load' is enabled (OTP-14612) on_load_trace_on_load(Config) -> + %% 'on_load' enables tracing for all newly loaded modules, so we make a dry + %% run to ensure that ancillary modules like 'merl' won't be loaded during + %% the actual test. + on_load_update(Config), + Papa = self(), Tracer = spawn_link(fun F() -> receive M -> Papa ! M end, F() end), {tracer,[]} = erlang:trace_info(self(),tracer), diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 5a8bbd56c4..8dd4ef1987 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -40,7 +40,8 @@ monitor_nodes_errors/1, monitor_nodes_combinations/1, monitor_nodes_cleanup/1, - monitor_nodes_many/1]). + monitor_nodes_many/1, + dist_ctrl_proc_smoke/1]). %% Performs the test at another node. -export([get_socket_priorities/0, @@ -52,7 +53,7 @@ -export([init_per_testcase/2, end_per_testcase/2]). --export([start_node/2]). +-export([dist_cntrlr_output_test/2]). -export([pinger/1]). @@ -67,10 +68,11 @@ suite() -> [{ct_hooks,[ts_install_cth]}, - {timetrap,{minutes,4}}]. + {timetrap,{minutes,12}}]. all() -> - [tick, tick_change, nodenames, hostnames, illegal_nodenames, + [dist_ctrl_proc_smoke, + tick, tick_change, nodenames, hostnames, illegal_nodenames, connect_node, hidden_node, setopts, table_waste, net_setuptime, inet_dist_options_options, @@ -116,10 +118,12 @@ connect_node(Config) when is_list(Config) -> ok. tick(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(erl_distribution_SUITE)), + run_dist_configs(fun tick/2, Config). +tick(DCfg, _Config) -> %% First check that the normal case is OK! - {ok, Node} = start_node(dist_test, "-pa " ++ PaDir), + [Name1, Name2] = get_nodenames(2, dist_test), + {ok, Node} = start_node(DCfg, Name1), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [node()]), erlang:monitor_node(Node, true), @@ -143,14 +147,12 @@ tick(Config) when is_list(Config) -> %% Set the ticktime on the server node to 100 secs so the server %% node doesn't tick the client node within the interval ... - {ok, ServNode} = start_node(dist_test_server, - "-kernel net_ticktime 100 " - "-pa " ++ PaDir), + {ok, ServNode} = start_node(DCfg, Name2, + "-kernel net_ticktime 100"), rpc:call(ServNode, erl_distribution_SUITE, tick_serv_test, [Node, node()]), - {ok, _} = start_node(dist_test, - "-kernel net_ticktime 12 " - "-pa " ++ PaDir), + {ok, Node} = start_node(DCfg, Name1, + "-kernel net_ticktime 12"), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [ServNode]), spawn_link(erl_distribution_SUITE, keep_conn, [Node]), @@ -180,6 +182,9 @@ tick(Config) when is_list(Config) -> %% Checks that pinging nonexistyent nodes does not waste space in distribution table. table_waste(Config) when is_list(Config) -> + run_dist_configs(fun table_waste/2, Config). + +table_waste(DCfg, _Config) -> {ok, HName} = inet:gethostname(), F = fun(0,_F) -> []; (N,F) -> @@ -189,7 +194,7 @@ table_waste(Config) when is_list(Config) -> F(N-1,F) end, F(256,F), - {ok, N} = start_node(erl_distribution_300,""), + {ok, N} = start_node(DCfg, erl_distribution_300), stop_node(N), ok. @@ -295,13 +300,16 @@ gethostname() -> %% Test that pinging an illegal nodename does not kill the node. illegal_nodenames(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(erl_distribution_SUITE)), - {ok, Node}=start_node(illegal_nodenames, "-pa " ++ PaDir), + run_dist_configs(fun illegal_nodenames/2, Config). + +illegal_nodenames(DCfg, _Config) -> + {ok, Node}=start_node(DCfg, illegal_nodenames), monitor_node(Node, true), RPid=rpc:call(Node, erlang, spawn, [?MODULE, pinger, [self()]]), receive {RPid, pinged} -> + monitor_node(Node, false), ok; {nodedown, Node} -> ct:fail("Remote node died.") @@ -318,22 +326,25 @@ pinger(Starter) -> %% Test that you can set the net_setuptime properly. net_setuptime(Config) when is_list(Config) -> + run_dist_configs(fun net_setuptime/2, Config). + +net_setuptime(DCfg, _Config) -> + %% In this test case, we reluctantly accept shorter times than the given %% setup time, because the connection attempt can end in a %% "Host unreachable" error before the timeout fires. - Res0 = do_test_setuptime("2"), + Res0 = do_test_setuptime(DCfg, "2"), io:format("Res0 = ~p", [Res0]), true = (Res0 =< 4000), - Res1 = do_test_setuptime("0.3"), + Res1 = do_test_setuptime(DCfg, "0.3"), io:format("Res1 = ~p", [Res1]), true = (Res1 =< 500), ok. -do_test_setuptime(Setuptime) when is_list(Setuptime) -> - PaDir = filename:dirname(code:which(?MODULE)), - {ok, Node} = start_node(dist_setuptime_test, "-pa " ++ PaDir ++ - " -kernel net_setuptime " ++ Setuptime), +do_test_setuptime(DCfg, Setuptime) when is_list(Setuptime) -> + {ok, Node} = start_node(DCfg, dist_setuptime_test, + "-kernel net_setuptime " ++ Setuptime), Res = rpc:call(Node,?MODULE,time_ping,[?DUMMY_NODE]), stop_node(Node), Res. @@ -399,32 +410,36 @@ tick_cli_test1(Node) -> end. setopts(Config) when is_list(Config) -> + run_dist_configs(fun setopts/2, Config). + +setopts(DCfg, _Config) -> register(setopts_regname, self()), [N1,N2,N3,N4] = get_nodenames(4, setopts), - {_N1F,Port1} = start_node_unconnected(N1, ?MODULE, run_remote_test, + {_N1F,Port1} = start_node_unconnected(DCfg, N1, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "1", "ping"]), 0 = wait_for_port_exit(Port1), - {_N2F,Port2} = start_node_unconnected(N2, ?MODULE, run_remote_test, + {_N2F,Port2} = start_node_unconnected(DCfg, N2, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "2", "ping"]), 0 = wait_for_port_exit(Port2), {ok, LSock} = gen_tcp:listen(0, [{packet,2}, {active,false}]), {ok, LTcpPort} = inet:port(LSock), - {N3F,Port3} = start_node_unconnected(N3, ?MODULE, run_remote_test, + {N3F,Port3} = start_node_unconnected(DCfg, N3, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "1", integer_to_list(LTcpPort)]), wait_and_connect(LSock, N3F, Port3), 0 = wait_for_port_exit(Port3), - {N4F,Port4} = start_node_unconnected(N4, ?MODULE, run_remote_test, + {N4F,Port4} = start_node_unconnected(DCfg, N4, ?MODULE, run_remote_test, ["setopts_do", atom_to_list(node()), "2", integer_to_list(LTcpPort)]), wait_and_connect(LSock, N4F, Port4), 0 = wait_for_port_exit(Port4), + unregister(setopts_regname), ok. wait_and_connect(LSock, NodeName, NodePort) -> @@ -518,9 +533,9 @@ opt_from_nr("2") -> {nodelay, false}. change_val(true) -> false; change_val(false) -> true. -start_node_unconnected(Name, Mod, Func, Args) -> +start_node_unconnected(DCfg, Name, Mod, Func, Args) -> FullName = full_node_name(Name), - CmdLine = mk_node_cmdline(Name,Mod,Func,Args), + CmdLine = mk_node_cmdline(DCfg, Name,Mod,Func,Args), io:format("Starting node ~p: ~s~n", [FullName, CmdLine]), case open_port({spawn, CmdLine}, [exit_status]) of Port when is_port(Port) -> @@ -534,7 +549,7 @@ full_node_name(PreName) -> atom_to_list(node())), list_to_atom(atom_to_list(PreName) ++ HostSuffix). -mk_node_cmdline(Name,Mod,Func,Args) -> +mk_node_cmdline(DCfg, Name,Mod,Func,Args) -> Static = "-noinput", Pa = filename:dirname(code:which(?MODULE)), Prog = case catch init:get_argument(progname) of @@ -551,6 +566,7 @@ mk_node_cmdline(Name,Mod,Func,Args) -> Prog ++ " " ++ Static ++ " " ++ NameSw ++ " " ++ NameStr + ++ " " ++ DCfg ++ " -pa " ++ Pa ++ " -env ERL_CRASH_DUMP " ++ Pwd ++ "/erl_crash_dump." ++ NameStr ++ " -setcookie " ++ atom_to_list(erlang:get_cookie()) @@ -560,7 +576,9 @@ mk_node_cmdline(Name,Mod,Func,Args) -> %% OTP-4255. tick_change(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(?MODULE)), + run_dist_configs(fun tick_change/2, Config). + +tick_change(DCfg, _Config) -> [BN, CN] = get_nodenames(2, tick_change), DefaultTT = net_kernel:get_net_ticktime(), unchanged = net_kernel:set_net_ticktime(DefaultTT, 60), @@ -577,14 +595,13 @@ tick_change(Config) when is_list(Config) -> end, wait_until(fun () -> 10 == net_kernel:get_net_ticktime() end), - {ok, B} = start_node(BN, "-kernel net_ticktime 10 -pa " ++ PaDir), - {ok, C} = start_node(CN, "-kernel net_ticktime 10 -hidden -pa " - ++ PaDir), + {ok, B} = start_node(DCfg, BN, "-kernel net_ticktime 10"), + {ok, C} = start_node(DCfg, CN, "-kernel net_ticktime 10 -hidden"), OTE = process_flag(trap_exit, true), case catch begin - run_tick_change_test(B, C, 10, 1, PaDir), - run_tick_change_test(B, C, 1, 10, PaDir) + run_tick_change_test(DCfg, B, C, 10, 1), + run_tick_change_test(DCfg, B, C, 1, 10) end of {'EXIT', Reason} -> stop_node(B), @@ -626,7 +643,7 @@ wait_for_nodedowns(Tester, Ref) -> end, wait_for_nodedowns(Tester, Ref). -run_tick_change_test(B, C, PrevTT, TT, PaDir) -> +run_tick_change_test(DCfg, B, C, PrevTT, TT) -> [DN, EN] = get_nodenames(2, tick_change), Tester = self(), @@ -640,8 +657,8 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> wait_for_nodedowns(Tester, Ref) end, - {ok, D} = start_node(DN, "-kernel net_ticktime " - ++ integer_to_list(PrevTT) ++ " -pa " ++ PaDir), + {ok, D} = start_node(DCfg, DN, "-kernel net_ticktime " + ++ integer_to_list(PrevTT)), NMA = spawn_link(fun () -> MonitorNodes([B, C, D]) end), NMB = spawn_link(B, fun () -> MonitorNodes([node(), C, D]) end), @@ -674,8 +691,8 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> sleep(7), change_initiated = rpc:call(C,net_kernel,set_net_ticktime,[TT,10]), - {ok, E} = start_node(EN, "-kernel net_ticktime " - ++ integer_to_list(TT) ++ " -pa " ++ PaDir), + {ok, E} = start_node(DCfg, EN, "-kernel net_ticktime " + ++ integer_to_list(TT)), NME = spawn_link(E, fun () -> MonitorNodes([node(), B, C, D]) end), NMA2 = spawn_link(fun () -> MonitorNodes([E]) end), NMB2 = spawn_link(B, fun () -> MonitorNodes([E]) end), @@ -735,12 +752,13 @@ run_tick_change_test(B, C, PrevTT, TT, PaDir) -> %% %% Basic test of hidden node. hidden_node(Config) when is_list(Config) -> - PaDir = filename:dirname(code:which(?MODULE)), - VArgs = "-pa " ++ PaDir, - HArgs = "-hidden -pa " ++ PaDir, - {ok, V} = start_node(visible_node, VArgs), + run_dist_configs(fun hidden_node/2, Config). + +hidden_node(DCfg, _Config) -> + HArgs = "-hidden", + {ok, V} = start_node(DCfg, visible_node), VMN = start_monitor_nodes_proc(V), - {ok, H} = start_node(hidden_node, HArgs), + {ok, H} = start_node(DCfg, hidden_node, HArgs), %% Connect visible_node -> hidden_node connect_nodes(V, H), test_nodes(V, H), @@ -748,9 +766,9 @@ hidden_node(Config) when is_list(Config) -> sleep(5), check_monitor_nodes_res(VMN, H), stop_node(V), - {ok, H} = start_node(hidden_node, HArgs), + {ok, H} = start_node(DCfg, hidden_node, HArgs), HMN = start_monitor_nodes_proc(H), - {ok, V} = start_node(visible_node, VArgs), + {ok, V} = start_node(DCfg, visible_node), %% Connect hidden_node -> visible_node connect_nodes(H, V), test_nodes(V, H), @@ -850,9 +868,9 @@ do_inet_dist_options_options(Prio) -> "-kernel inet_dist_connect_options "++PriorityString++" " "-kernel inet_dist_listen_options "++PriorityString, {ok,Node1} = - start_node(inet_dist_options_1, InetDistOptions), + start_node("", inet_dist_options_1, InetDistOptions), {ok,Node2} = - start_node(inet_dist_options_2, InetDistOptions), + start_node("", inet_dist_options_2, InetDistOptions), %% pong = rpc:call(Node1, net_adm, ping, [Node2]), @@ -885,6 +903,9 @@ get_socket_priorities() -> %% monitor_nodes_nodedown_reason(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_nodedown_reason/2, Config). + +monitor_nodes_nodedown_reason(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [nodedown_reason]), @@ -892,10 +913,10 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> Names = get_numbered_nodenames(5, node), [NN1, NN2, NN3, NN4, NN5] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2), - {ok, N3} = start_node(NN3), - {ok, N4} = start_node(NN4, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2), + {ok, N3} = start_node(DCfg, NN3), + {ok, N4} = start_node(DCfg, NN4, "-hidden"), receive {nodeup, N1} -> ok end, receive {nodeup, N2} -> ok end, @@ -925,7 +946,7 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [nodedown_reason]), - {ok, N5} = start_node(NN5), + {ok, N5} = start_node(DCfg, NN5), stop_node(N5), receive {nodeup, N5} -> ok end, @@ -938,11 +959,14 @@ monitor_nodes_nodedown_reason(Config) when is_list(Config) -> monitor_nodes_complex_nodedown_reason(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_complex_nodedown_reason/2, Config). + +monitor_nodes_complex_nodedown_reason(DCfg, _Config) -> MonNodeState = monitor_node_state(), Me = self(), ok = net_kernel:monitor_nodes(true, [nodedown_reason]), [Name] = get_nodenames(1, monitor_nodes_complex_nodedown_reason), - {ok, Node} = start_node(Name, ""), + {ok, Node} = start_node(DCfg, Name, ""), Pid = spawn(Node, fun() -> Me ! {stuff, @@ -981,16 +1005,19 @@ monitor_nodes_complex_nodedown_reason(Config) when is_list(Config) -> %% monitor_nodes_node_type(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_node_type/2, Config). + +monitor_nodes_node_type(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [{node_type, all}]), Names = get_numbered_nodenames(9, node), [NN1, NN2, NN3, NN4, NN5, NN6, NN7, NN8, NN9] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2), - {ok, N3} = start_node(NN3, "-hidden"), - {ok, N4} = start_node(NN4, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2), + {ok, N3} = start_node(DCfg, NN3, "-hidden"), + {ok, N4} = start_node(DCfg, NN4, "-hidden"), receive {nodeup, N1} -> ok end, receive {nodeup, N2} -> ok end, @@ -1014,15 +1041,15 @@ monitor_nodes_node_type(Config) when is_list(Config) -> receive {nodedown, N4, [{node_type, hidden}]} -> ok end, ok = net_kernel:monitor_nodes(false, [{node_type, all}]), - {ok, N5} = start_node(NN5), + {ok, N5} = start_node(DCfg, NN5), receive {nodeup, N5} -> ok end, stop_node(N5), receive {nodedown, N5} -> ok end, ok = net_kernel:monitor_nodes(true, [{node_type, hidden}]), - {ok, N6} = start_node(NN6), - {ok, N7} = start_node(NN7, "-hidden"), + {ok, N6} = start_node(DCfg, NN6), + {ok, N7} = start_node(DCfg, NN7, "-hidden"), receive {nodeup, N6} -> ok end, @@ -1037,8 +1064,8 @@ monitor_nodes_node_type(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [{node_type, hidden}]), ok = net_kernel:monitor_nodes(false), - {ok, N8} = start_node(NN8), - {ok, N9} = start_node(NN9, "-hidden"), + {ok, N8} = start_node(DCfg, NN8), + {ok, N9} = start_node(DCfg, NN9, "-hidden"), receive {nodeup, N8, [{node_type, visible}]} -> ok end, stop_node(N8), @@ -1058,6 +1085,9 @@ monitor_nodes_node_type(Config) when is_list(Config) -> %% monitor_nodes_misc(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_misc/2, Config). + +monitor_nodes_misc(DCfg, _Config) -> MonNodeState = monitor_node_state(), ok = net_kernel:monitor_nodes(true), ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), @@ -1065,8 +1095,8 @@ monitor_nodes_misc(Config) when is_list(Config) -> Names = get_numbered_nodenames(3, node), [NN1, NN2, NN3] = Names, - {ok, N1} = start_node(NN1), - {ok, N2} = start_node(NN2, "-hidden"), + {ok, N1} = start_node(DCfg, NN1), + {ok, N2} = start_node(DCfg, NN2, "-hidden"), receive {nodeup, N1} -> ok end, @@ -1092,7 +1122,7 @@ monitor_nodes_misc(Config) when is_list(Config) -> ok = net_kernel:monitor_nodes(false, [{node_type, all}, nodedown_reason]), - {ok, N3} = start_node(NN3), + {ok, N3} = start_node(DCfg, NN3), receive {nodeup, N3} -> ok end, stop_node(N3), receive {nodedown, N3} -> ok end, @@ -1107,15 +1137,18 @@ monitor_nodes_misc(Config) when is_list(Config) -> %% messages from Node and that {nodedown, Node} messages are %% received after messages from Node. monitor_nodes_otp_6481(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_otp_6481/2, Config). + +monitor_nodes_otp_6481(DCfg, Config) -> io:format("Testing nodedown...~n"), - monitor_nodes_otp_6481_test(Config, nodedown), + monitor_nodes_otp_6481_test(DCfg, Config, nodedown), io:format("ok~n"), io:format("Testing nodeup...~n"), - monitor_nodes_otp_6481_test(Config, nodeup), + monitor_nodes_otp_6481_test(DCfg, Config, nodeup), io:format("ok~n"), ok. -monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> +monitor_nodes_otp_6481_test(DCfg, Config, TestType) when is_list(Config) -> MonNodeState = monitor_node_state(), NodeMsg = make_ref(), Me = self(), @@ -1164,7 +1197,7 @@ monitor_nodes_otp_6481_test(Config, TestType) when is_list(Config) -> end ++ MonNodeState, - {ok, Node} = start_node(Name, "", this), + {ok, Node} = start_node(DCfg, Name, "", this), receive {nodeup, Node} -> ok end, RemotePid = spawn(Node, @@ -1249,17 +1282,20 @@ monitor_nodes_errors(Config) when is_list(Config) -> ok. monitor_nodes_combinations(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_combinations/2, Config). + +monitor_nodes_combinations(DCfg, _Config) -> MonNodeState = monitor_node_state(), monitor_nodes_all_comb(true), [VisibleName, HiddenName] = get_nodenames(2, monitor_nodes_combinations), - {ok, Visible} = start_node(VisibleName, ""), + {ok, Visible} = start_node(DCfg, VisibleName, ""), receive_all_comb_nodeup_msgs(visible, Visible), no_msgs(), stop_node(Visible), receive_all_comb_nodedown_msgs(visible, Visible, connection_closed), no_msgs(), - {ok, Hidden} = start_node(HiddenName, "-hidden"), + {ok, Hidden} = start_node(DCfg, HiddenName, "-hidden"), receive_all_comb_nodeup_msgs(hidden, Hidden), no_msgs(), stop_node(Hidden), @@ -1395,6 +1431,9 @@ monitor_nodes_cleanup(Config) when is_list(Config) -> ok. monitor_nodes_many(Config) when is_list(Config) -> + run_dist_configs(fun monitor_nodes_many/2, Config). + +monitor_nodes_many(DCfg, _Config) -> MonNodeState = monitor_node_state(), [Name] = get_nodenames(1, monitor_nodes_many), %% We want to perform more than 2^16 net_kernel:monitor_nodes @@ -1402,7 +1441,7 @@ monitor_nodes_many(Config) when is_list(Config) -> No = (1 bsl 16) + 17, repeat(fun () -> ok = net_kernel:monitor_nodes(true) end, No), No = length(monitor_node_state()) - length(MonNodeState), - {ok, Node} = start_node(Name), + {ok, Node} = start_node(DCfg, Name), repeat(fun () -> receive {nodeup, Node} -> ok end end, No), stop_node(Node), repeat(fun () -> receive {nodedown, Node} -> ok end end, No), @@ -1411,8 +1450,118 @@ monitor_nodes_many(Config) when is_list(Config) -> MonNodeState = monitor_node_state(), ok. +dist_ctrl_proc_smoke(Config) when is_list(Config) -> + ThisNode = node(), + [Name1, Name2] = get_nodenames(2, dist_ctrl_proc_example_smoke), + GetSizeArg = " -gen_tcp_dist_output_loop " + ++ atom_to_list(?MODULE) ++ " " + ++ "dist_cntrlr_output_test", + {ok, Node1} = start_node("", Name1, "-proto_dist gen_tcp"), + {ok, Node2} = start_node("", Name2, "-proto_dist gen_tcp" ++ GetSizeArg), + pong = rpc:call(Node1, net_adm, ping, [Node2]), + NL1 = lists:sort([ThisNode, Node2]), + NL2 = lists:sort([ThisNode, Node1]), + NL1 = lists:sort(rpc:call(Node1, erlang, nodes, [])), + NL2 = lists:sort(rpc:call(Node2, erlang, nodes, [])), + + %% Verify that we actually are executing the distribution + %% module we expect and also massage message passing over + %% it a bit... + Ps1 = rpc:call(Node1, erlang, processes, []), + try + lists:foreach( + fun (P) -> + case rpc:call(Node1, erlang, process_info, [P, current_stacktrace]) of + undefined -> + ok; + {current_stacktrace, StkTrace} -> + lists:foreach(fun ({gen_tcp_dist, + dist_cntrlr_output_loop, + 2, _}) -> + io:format("~p ~p~n", [P, StkTrace]), + throw(found_it); + (_) -> + ok + end, StkTrace) + end + end, Ps1), + exit({missing, dist_cntrlr_output_loop}) + catch + throw:found_it -> ok + end, + + Ps2 = rpc:call(Node2, erlang, processes, []), + try + lists:foreach( + fun (P) -> + case rpc:call(Node2, erlang, process_info, [P, current_stacktrace]) of + undefined -> + ok; + {current_stacktrace, StkTrace} -> + lists:foreach(fun ({erl_distribution_SUITE, + dist_cntrlr_output_loop, + 2, _}) -> + io:format("~p ~p~n", [P, StkTrace]), + throw(found_it); + (_) -> + ok + end, StkTrace) + end + end, Ps2), + exit({missing, dist_cntrlr_output_loop}) + catch + throw:found_it -> ok + end, + + stop_node(Node1), + stop_node(Node2), + ok. + %% Misc. functions +run_dist_configs(Func, Config) -> + GetSizeArg = " -gen_tcp_dist_output_loop " + ++ atom_to_list(?MODULE) ++ " " + ++ "dist_cntrlr_output_test", + lists:map(fun ({DCfgName, DCfg}) -> + io:format("~n~n=== Running ~s configuration ===~n~n", + [DCfgName]), + Func(DCfg, Config) + end, + [{"default", ""}, + {"gen_tcp_dist", "-proto_dist gen_tcp"}, + {"gen_tcp_dist (get_size)", "-proto_dist gen_tcp" ++ GetSizeArg}]). + +dist_cntrlr_output_test(DHandle, Socket) -> + false = erlang:dist_ctrl_get_opt(DHandle, get_size), + false = erlang:dist_ctrl_set_opt(DHandle, get_size, true), + true = erlang:dist_ctrl_get_opt(DHandle, get_size), + true = erlang:dist_ctrl_set_opt(DHandle, get_size, false), + false = erlang:dist_ctrl_get_opt(DHandle, get_size), + false = erlang:dist_ctrl_set_opt(DHandle, get_size, true), + true = erlang:dist_ctrl_get_opt(DHandle, get_size), + dist_cntrlr_output_loop(DHandle, Socket). + +dist_cntrlr_send_data(DHandle, Socket) -> + case erlang:dist_ctrl_get_data(DHandle) of + none -> + erlang:dist_ctrl_get_data_notification(DHandle); + {Size, Data} -> + Size = erlang:iolist_size(Data), + ok = gen_tcp:send(Socket, Data), + dist_cntrlr_send_data(DHandle, Socket) + end. + +dist_cntrlr_output_loop(DHandle, Socket) -> + receive + dist_data -> + %% Outgoing data from this node... + dist_cntrlr_send_data(DHandle, Socket); + _ -> + ok %% Drop garbage message... + end, + dist_cntrlr_output_loop(DHandle, Socket). + monitor_node_state() -> erts_debug:set_internal_state(available_internal_state, true), MonitoringNodes = erts_debug:get_internal_state(monitoring_nodes), @@ -1438,25 +1587,25 @@ print_my_messages() -> sleep(T) -> receive after T * 1000 -> ok end. -start_node(Name, Param, this) -> +start_node(DCfg, Name, Param, this) -> NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), test_server:start_node(Name, peer, [{args, NewParam}, {erl, [this]}]); -start_node(Name, Param, "this") -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, "this") -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [this]}]); -start_node(Name, Param, Rel) when is_atom(Rel) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, Rel) when is_atom(Rel) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [{release, atom_to_list(Rel)}]}]); -start_node(Name, Param, Rel) when is_list(Rel) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param, Rel) when is_list(Rel) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, peer, [{args, NewParam}, {erl, [{release, Rel}]}]). -start_node(Name, Param) -> - NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)), +start_node(DCfg, Name, Param) -> + NewParam = Param ++ " -pa " ++ filename:dirname(code:which(?MODULE)) ++ " " ++ DCfg, test_server:start_node(Name, slave, [{args, NewParam}]). -start_node(Name) -> - start_node(Name, ""). +start_node(DCfg, Name) -> + start_node(DCfg, Name, ""). stop_node(Node) -> test_server:stop_node(Node). diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index a51025cba6..e095e589a3 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -2191,6 +2191,9 @@ unc_paths(Config) when is_list(Config) -> {ok, _} = file:read_file_info("C:\\Windows\\explorer.exe"), {ok, _} = file:read_file_info("\\\\localhost\\c$\\Windows\\explorer.exe"), + {ok, Files} = file:list_dir("C:\\Windows\\"), + {ok, Files} = file:list_dir("\\\\localhost\\c$\\Windows\\"), + {ok, Cwd} = file:get_cwd(), try @@ -3741,19 +3744,33 @@ otp_10852(Config) when is_list(Config) -> ok = rpc_call(Node, read_file, [B]), ok = rpc_call(Node, make_link, [B,B]), case rpc_call(Node, make_symlink, [B,B]) of - ok -> ok; - {error, E} when (E =:= enotsup) or (E =:= eperm) -> - {win32,_} = os:type() + {error, eilseq} -> + %% Some versions of OS X refuse to create files with illegal names. + {unix,darwin} = os:type(); + {error, eperm} -> + %% The test user might not have permission to create symlinks. + {win32,_} = os:type(); + ok -> + ok end, ok = rpc_call(Node, delete, [B]), - ok = rpc_call(Node, make_dir, [B]), + case rpc_call(Node, make_dir, [B]) of + {error, eilseq} -> + {unix,darwin} = os:type(); + ok -> + ok + end, ok = rpc_call(Node, del_dir, [B]), - ok = rpc_call(Node, write_file, [B,B]), - {ok, Fd} = rpc_call(Node, open, [B,[read]]), - ok = rpc_call(Node, close, [Fd]), - {ok,0} = rpc_call(Node, copy, [B,B]), - {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), - ok = rpc_call(Node, close, [Fd2]), + case rpc_call(Node, write_file, [B,B]) of + {error, eilseq} -> + {unix,darwin} = os:type(); + ok -> + {ok, Fd} = rpc_call(Node, open, [B,[read]]), + ok = rpc_call(Node, close, [Fd]), + {ok,0} = rpc_call(Node, copy, [B,B]), + {ok, Fd2, B} = rpc_call(Node, path_open, [["."], B, [read]]), + ok = rpc_call(Node, close, [Fd2]) + end, true = test_server:stop_node(Node), ok. diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl index 3afc647081..26cfd187c7 100644 --- a/lib/kernel/test/file_name_SUITE.erl +++ b/lib/kernel/test/file_name_SUITE.erl @@ -632,10 +632,13 @@ make_icky_dir(Mod, IckyDirName) -> hopeless_darwin() -> case {os:type(),os:version()} of - {{unix,darwin},{Major,_,_}} when Major < 9 -> - true; - _ -> - false + {{unix,darwin},{Major,_,_}} -> + %% icky file names worked between 10 and 17, but started returning + %% EILSEQ in 18. The check against 18 is exact in case newer + %% versions of Darwin support them again. + Major < 9 orelse Major =:= 18; + _ -> + false end. make_very_icky_dir(Mod, DirName) -> diff --git a/lib/kernel/test/gen_sctp_SUITE.erl b/lib/kernel/test/gen_sctp_SUITE.erl index a0ae792ba9..e4c489bd10 100644 --- a/lib/kernel/test/gen_sctp_SUITE.erl +++ b/lib/kernel/test/gen_sctp_SUITE.erl @@ -1459,11 +1459,11 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo) -> do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun). %% do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> - ServerFamily = get_family_by_addrs(ServerAddresses), + {ServerFamily, ServerOpts} = get_family_by_addrs(ServerAddresses), io:format("Serving ~p addresses: ~p~n", [ServerFamily, ServerAddresses]), S1 = ok(gen_sctp:open(0, [{ip,Addr} || Addr <- ServerAddresses] ++ - [ServerFamily])), + [ServerFamily|ServerOpts])), ok = gen_sctp:listen(S1, true), P1 = ok(inet:port(S1)), ClientFamily = get_family_by_addr(AddressToConnectTo), @@ -1493,9 +1493,9 @@ do_open_and_connect(ServerAddresses, AddressToConnectTo, Fun) -> %% If at least one of the addresses is an ipv6 address, return inet6, else inet. get_family_by_addrs(Addresses) -> case lists:usort([get_family_by_addr(Addr) || Addr <- Addresses]) of - [inet, inet6] -> inet6; - [inet] -> inet; - [inet6] -> inet6 + [inet, inet6] -> {inet6, [{ipv6_v6only, false}]}; + [inet] -> {inet, []}; + [inet6] -> {inet6, []} end. get_family_by_addr(Addr) when tuple_size(Addr) =:= 4 -> inet; diff --git a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c index b91dca61d4..96938f9071 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c +++ b/lib/kernel/test/gen_tcp_api_SUITE_data/gen_tcp_api_SUITE.c @@ -30,6 +30,7 @@ #define sock_close(s) closesocket(s) #else #include <sys/socket.h> +#include <unistd.h> #define sock_close(s) close(s) #endif diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 52edfaee29..edf30448c4 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -2086,8 +2086,39 @@ test_pktoptions(Family, Spec, CheckConnect, OSType, OSVer) -> %%% {ok,<<"hi">>} = gen_tcp:recv(S1, 2, Timeout), %% %% Verify returned remote options - {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), - {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), + VerifyRemOpts = + fun(S, Role) -> + case inet:getopts(S, [pktoptions]) of + {ok, [{pktoptions, PktOpts1}]} -> + PktOpts1; + {ok, UnexpOK1} -> + io:format("Unexpected OK (~w): " + "~n ~p" + "~n", [Role, UnexpOK1]), + exit({unexpected_getopts_ok, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpOK1}); + {error, UnexpERR1} -> + io:format("Unexpected ERROR (~w): " + "~n ~p" + "~n", [Role, UnexpERR1]), + exit({unexpected_getopts_failure, + Role, + Spec, + TrueRecvOpts, + OptsVals, + OptsValsDefault, + UnexpERR1}) + end + end, + OptsVals1 = VerifyRemOpts(S1, dest), + OptsVals2 = VerifyRemOpts(S2, orig), + %% {ok,[{pktoptions,OptsVals1}]} = inet:getopts(S1, [pktoptions]), + %% {ok,[{pktoptions,OptsVals2}]} = inet:getopts(S2, [pktoptions]), (Result1 = sets_eq(OptsVals1, OptsVals)) orelse io:format( "Accept differs: ~p neq ~p~n", [OptsVals1,OptsVals]), @@ -3430,7 +3461,7 @@ wait(Mref) -> %% OTP-15536 %% Test that send error works correctly for delay_send -delay_send_error(Config) -> +delay_send_error(_Config) -> {ok, LS} = gen_tcp:listen(0, [{reuseaddr, true}, {packet, 1}, {active, false}]), {ok,{{0,0,0,0},PortNum}}=inet:sockname(LS), P = spawn_link( diff --git a/lib/kernel/test/logger_disk_log_h_SUITE.erl b/lib/kernel/test/logger_disk_log_h_SUITE.erl index 13b30835a1..36d98eaa25 100644 --- a/lib/kernel/test/logger_disk_log_h_SUITE.erl +++ b/lib/kernel/test/logger_disk_log_h_SUITE.erl @@ -1243,12 +1243,12 @@ restart_after(Config) -> {Log,HConfig,DLHConfig} = start_handler(?MODULE, ?FUNCTION_NAME, Config), NewHConfig1 = HConfig#{config=>DLHConfig#{overload_kill_enable=>true, - overload_kill_qlen=>10, + overload_kill_qlen=>4, overload_kill_restart_after=>infinity}}, ok = logger:update_handler_config(?MODULE, NewHConfig1), MRef1 = erlang:monitor(process, whereis(h_proc_name())), %% kill handler - send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), + send_burst({n,100}, {spawn,5,0}, {chars,79}, notice), receive {'DOWN', MRef1, _, _, _Reason1} -> file_delete(Log), @@ -1265,13 +1265,13 @@ restart_after(Config) -> RestartAfter = ?OVERLOAD_KILL_RESTART_AFTER, NewHConfig2 = HConfig#{config=>DLHConfig#{overload_kill_enable=>true, - overload_kill_qlen=>10, + overload_kill_qlen=>4, overload_kill_restart_after=>RestartAfter}}, ok = logger:update_handler_config(?MODULE, NewHConfig2), Pid0 = whereis(h_proc_name()), MRef2 = erlang:monitor(process, Pid0), %% kill handler - send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), + send_burst({n,100}, {spawn,5,0}, {chars,79}, notice), receive {'DOWN', MRef2, _, _, _Reason2} -> file_delete(Log), diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 7bebe1ba70..b1ae513223 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.3 +KERNEL_VSN = 6.3.1 diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 77afb8250c..02bc884e36 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -160,7 +160,7 @@ {'sync_transaction', Retries::non_neg_integer()}. -type table() :: atom(). -type storage_type() :: 'ram_copies' | 'disc_copies' | 'disc_only_copies'. --type index_attr() :: atom() | non_neg_integer(). +-type index_attr() :: atom() | non_neg_integer() | {atom()}. -type write_locks() :: 'write' | 'sticky_write'. -type read_locks() :: 'read'. -type lock_kind() :: write_locks() | read_locks(). @@ -1277,6 +1277,14 @@ match_object(Tid, Ts, Tab, Pat, LockKind) match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> abort({bad_type, Tab, Pat}). +add_written_index(Store, Pos, Tab, Key, Objs) when is_integer(Pos) -> + Pat = setelement(Pos, val({Tab, wild_pattern}), Key), + add_written_match(Store, Pat, Tab, Objs); +add_written_index(Store, Pos, Tab, Key, Objs) when is_tuple(Pos) -> + IxF = mnesia_index:index_vals_f(val({Tab, storage_type}), Tab, Pos), + Ops = find_ops(Store, Tab, '_'), + add_ix_match(Ops, Objs, IxF, Key, val({Tab, setorbag})). + add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), @@ -1303,6 +1311,46 @@ add_match([{_Oid, Val, write}|R], Objs, bag) -> add_match([{Oid, Val, write}|R], Objs, set) -> add_match(R, [Val | deloid(Oid,Objs)],set). +add_ix_match([], Objs, _IxF, _Key, _Type) -> + Objs; +add_ix_match(Written, Objs, IxF, Key, ordered_set) -> + %% Must use keysort which is stable + add_ordered_match(lists:keysort(1, ix_filter_ops(IxF, Key, Written)), Objs, []); +add_ix_match([{Oid, _, delete}|R], Objs, IxF, Key, Type) -> + add_ix_match(R, deloid(Oid, Objs), IxF, Key, Type); +add_ix_match([{_Oid, Val, delete_object}|R], Objs, IxF, Key, Type) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, lists:delete(Val, Objs), IxF, Key, Type); + false -> + add_ix_match(R, Objs, IxF, Key, Type) + end; +add_ix_match([{_Oid, Val, write}|R], Objs, IxF, Key, bag) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | lists:delete(Val, Objs)], IxF, Key, bag); + false -> + add_ix_match(R, Objs, IxF, Key, bag) + end; +add_ix_match([{Oid, Val, write}|R], Objs, IxF, Key, set) -> + case ix_match(Val, IxF, Key) of + true -> + add_ix_match(R, [Val | deloid(Oid,Objs)],IxF,Key,set); + false -> + add_ix_match(R, Objs, IxF, Key, set) + end. + +ix_match(Val, IxF, Key) -> + lists:member(Key, IxF(Val)). + +ix_filter_ops(IxF, Key, Ops) -> + lists:filter( + fun({_Oid, Obj, write}) -> + ix_match(Obj, IxF, Key); + (_) -> + true + end, Ops). + %% For ordered_set only !! add_ordered_match(Written = [{{_, Key}, _, _}|_], [Obj|Objs], Acc) when Key > element(2, Obj) -> @@ -1641,6 +1689,16 @@ index_match_object(Tid, Ts, Tab, Pat, Attr, LockKind) dirty_index_match_object(Tab, Pat, Attr); % Should be optimized? tid -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + case LockKind of + read -> + Store = Ts#tidstore.store, + mnesia_locker:rlock_table(Tid, Store, Tab), + Objs = dirty_match_object(Tab, Pat), + add_written_match(Store, Pat, Tab, Objs); + _ -> + abort({bad_type, Tab, LockKind}) + end; Pos when Pos =< tuple_size(Pat) -> case LockKind of read -> @@ -1688,8 +1746,8 @@ index_read(Tid, Ts, Tab, Key, Attr, LockKind) false -> Store = Ts#tidstore.store, Objs = mnesia_index:read(Tid, Store, Tab, Key, Pos), - Pat = setelement(Pos, val({Tab, wild_pattern}), Key), - add_written_match(Store, Pat, Tab, Objs); + add_written_index( + Ts#tidstore.store, Pos, Tab, Key, Objs); true -> abort({bad_type, Tab, Attr, Key}) end; @@ -1825,7 +1883,7 @@ remote_dirty_match_object(Tab, Pat) -> false -> mnesia_lib:db_match_object(Tab, Pat); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_match_object(Tab, Pat, PosList) end. @@ -1857,7 +1915,7 @@ remote_dirty_select(Tab, Spec) -> false -> mnesia_lib:db_select(Tab, Spec); true -> - PosList = val({Tab, index}), + PosList = regular_indexes(Tab), remote_dirty_select(Tab, Spec, PosList) end; _ -> @@ -1924,6 +1982,8 @@ dirty_index_match_object(Pat, _Attr) -> dirty_index_match_object(Tab, Pat, Attr) when is_atom(Tab), Tab /= schema, is_tuple(Pat), tuple_size(Pat) > 2 -> case mnesia_schema:attr_tab_to_pos(Tab, Attr) of + {_} -> + dirty_match_object(Tab, Pat); Pos when Pos =< tuple_size(Pat) -> case has_var(element(2, Pat)) of false -> @@ -3254,3 +3314,7 @@ put_activity_id(Activity) -> mnesia_tm:put_activity_id(Activity). put_activity_id(Activity,Fun) -> mnesia_tm:put_activity_id(Activity,Fun). + +regular_indexes(Tab) -> + PosList = val({Tab, index}), + [P || P <- PosList, is_integer(P)]. diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 098265d5fc..6f1c21e3b9 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,8 +1,8 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 1996-2018. All Rights Reserved. -%% +%% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at @@ -14,7 +14,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%% +%% %% %CopyrightEnd% %% @@ -37,7 +37,7 @@ db_match_erase/2, get_index_table/2, get_index_table/3, - + tab2filename/2, init_index/2, init_indecies/3, @@ -45,6 +45,7 @@ del_transient/3, del_index_table/3, + index_vals_f/3, index_info/2, ext_index_instances/1]). @@ -60,9 +61,14 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList = mnesia_locker:ixrlock(Tid, Store, Tab, IxKey, Pos), %% Remove all tuples which don't include Ixkey, happens when Tab is a bag case val({Tab, setorbag}) of - bag -> + bag when is_integer(Pos) -> mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> + bag when is_tuple(Pos) -> + TabStorage = val({Tab, storage_type}), + ValsF = index_vals_f(TabStorage, Tab, Pos), + [Obj || Obj <- ResList, + lists:member(IxKey, ValsF(Obj))]; + _ -> ResList end. @@ -136,7 +142,7 @@ del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> ValsF = index_vals_f(Storage, Tab, Pos), case SoB of - bag -> + bag -> del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table del_ixes(Type, Ixt, ValsF, Obj, K) @@ -197,7 +203,7 @@ merge([], _, _, Ack) -> realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... - + dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec @@ -233,7 +239,7 @@ dirty_read2(Tab, IxKey, Pos) -> end, Acc, mnesia_lib:db_get(Storage, Tab, K)) end, [], Keys)). -pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> +pick_index([{{{Pfx,_,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> {IxType, Ixt}; pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> {IxType, Ixt}; @@ -242,7 +248,7 @@ pick_index([_|T], Tab, Pos) -> pick_index([], Tab, Pos) -> mnesia:abort({no_exist, Tab, {index, Pos}}). - + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table @@ -387,12 +393,12 @@ init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> create_fun(Cont, Tab, Pos) -> IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> - Data = + Data = case Cont of {start, KeysPerChunk} -> mnesia_lib:db_init_chunk( disc_only_copies, Tab, KeysPerChunk); - '$end_of_table' -> + '$end_of_table' -> '$end_of_table'; _Else -> mnesia_lib:db_chunk(disc_only_copies, Cont) @@ -462,7 +468,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> %% Check later if mnesia_tm is sensitive about the order mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here @@ -470,7 +476,7 @@ add_index_info(Tab, SetOrBag, IxElem) -> mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. @@ -488,19 +494,19 @@ del_index_info(Tab, Pos) -> element(1,P)=/=Pos end, Old#index.pos_list) of - [] -> + [] -> IndexInfo = index_info(Old#index.setorbag,[]), mnesia_lib:set({Tab, index_info}, IndexInfo), mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, mnesia_lib:set({Tab, index_info}, Index), mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), - mnesia_lib:set({Tab, commit_work}, + mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end end. @@ -537,7 +543,7 @@ db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). - + db_select({ram, Ixt}, Pat) -> ets:select(Ixt, Pat); db_select({{ext,_,_} = Ext, Ixt}, Pat) -> @@ -545,7 +551,7 @@ db_select({{ext,_,_} = Ext, Ixt}, Pat) -> db_select({dets, Ixt}, Pat) -> dets:select(Ixt, Pat). - + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index 5b61b1af65..b43bc82801 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -53,7 +53,8 @@ MODULES= \ mnesia_measure_test \ mnesia_cost \ mnesia_dbn_meters \ - ext_test + ext_test \ + mnesia_index_plugin_test DocExamplesDir := ../doc/src/ diff --git a/lib/mnesia/test/mnesia_SUITE.erl b/lib/mnesia/test/mnesia_SUITE.erl index 24c1def6da..b41bf22efa 100644 --- a/lib/mnesia/test/mnesia_SUITE.erl +++ b/lib/mnesia/test/mnesia_SUITE.erl @@ -69,12 +69,13 @@ groups() -> %% covered. [{light, [], [{group, install}, {group, nice}, {group, evil}, - {group, mnesia_frag_test, light}, {group, qlc}, + {group, mnesia_frag_test, light}, {group, qlc}, {group, index_plugins}, {group, registry}, {group, config}, {group, examples}]}, {install, [], [{mnesia_install_test, all}]}, {nice, [], [{mnesia_nice_coverage_test, all}]}, {evil, [], [{mnesia_evil_coverage_test, all}]}, {qlc, [], [{mnesia_qlc_test, all}]}, + {index_plugins, [], [{mnesia_index_plugin_test, all}]}, {registry, [], [{mnesia_registry_test, all}]}, {config, [], [{mnesia_config_test, all}]}, {examples, [], [{mnesia_examples_test, all}]}, diff --git a/lib/mnesia/test/mnesia_index_plugin_test.erl b/lib/mnesia/test/mnesia_index_plugin_test.erl new file mode 100644 index 0000000000..44fe047c50 --- /dev/null +++ b/lib/mnesia/test/mnesia_index_plugin_test.erl @@ -0,0 +1,261 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_index_plugin_test). +-author('[email protected]'). + +-export([init_per_testcase/2, end_per_testcase/2, + init_per_group/2, end_per_group/2, + init_per_suite/1, end_per_suite/1, + all/0, groups/0]). + +-export([ + add_rm_plugin/1, + tab_with_plugin_index/1, + tab_with_multiple_plugin_indexes/1, + ix_match_w_plugin/1, + ix_match_w_plugin_ordered/1, + ix_match_w_plugin_bag/1 + ]). + +-export([ix_prefixes/3, % test plugin + ix_prefixes2/3]). % test plugin 2 + +-include("mnesia_test_lib.hrl"). + +init_per_suite(Conf) -> + Conf. + +end_per_suite(Conf) -> + Conf. + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +end_per_testcase(Func, Conf) -> + mnesia_test_lib:end_per_testcase(Func, Conf). + +all() -> + [add_rm_plugin, + tab_with_plugin_index, + tab_with_multiple_plugin_indexes, + ix_match_w_plugin, + ix_match_w_plugin_ordered, + ix_match_w_plugin_bag]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +add_rm_plugin(suite) -> []; +add_rm_plugin(Config) when is_list(Config) -> + [N1, N2] = Nodes = ?acquire_nodes(2, Config), + ok = add_plugin(), + ok = rpc_check_plugin(N1), + ok = rpc_check_plugin(N2), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ok = add_plugin(), + ok = add_plugin2(), + ok = del_plugin(), + ok = del_plugin2(), + ?verify_mnesia(Nodes, []). + +-define(PLUGIN1, {{pfx},?MODULE,ix_prefixes}). +-define(PLUGIN2, {{pfx2},?MODULE,ix_prefixes2}). + +add_plugin() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx}, ?MODULE, ix_prefixes), + [?PLUGIN1] = mnesia_schema:index_plugins(), + ok. + +add_plugin2() -> + {atomic, ok} = mnesia_schema:add_index_plugin({pfx2}, ?MODULE, ix_prefixes2), + [?PLUGIN1, ?PLUGIN2] = lists:sort(mnesia_schema:index_plugins()), + ok. + +del_plugin() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx}), + [?PLUGIN2] = mnesia_schema:index_plugins(), + ok. + +del_plugin2() -> + {atomic, ok} = mnesia_schema:delete_index_plugin({pfx2}), + [] = mnesia_schema:index_plugins(), + ok. + +rpc_check_plugin(N) -> + [?PLUGIN1] = + rpc:call(N, mnesia_schema, index_plugins, []), + ok. + +tab_with_plugin_index(suite) -> []; +tab_with_plugin_index(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(t, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}, + v2]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({t, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{t,1,a,"123"},{t,2,b,"12345"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + [{t,3,c,"6789"}] = + mnesia:dirty_index_read(t,"6789",v2), + [{t,1,a,"123"}] = + mnesia:dirty_match_object({t,'_',a,"123"}), + [{t,1,a,"123"}] = + mnesia:dirty_select(t, [{ {t,'_',a,"123"}, [], ['$_']}]), + mnesia:dirty_delete(t,2), + [{t,1,a,"123"}] = + mnesia:dirty_index_read(t,<<"123">>,{pfx}), + ?verify_mnesia(Nodes, []). + +tab_with_multiple_plugin_indexes(suite) -> []; +tab_with_multiple_plugin_indexes(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + ok = add_plugin2(), + {atomic, ok} = + mnesia:create_table(u, [{attributes, [k,v1,v2]}, + {index, [{{pfx}, ordered}, + {{pfx2}, ordered}]}]), + [ok,ok,ok,ok] = + [mnesia:dirty_write({u, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"6789"}, + {4,d,nil}]], + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"123">>,{pfx}), + [{u,1,a,"123"},{u,2,b,"12345"}] = + mnesia:dirty_index_read(u,<<"321">>,{pfx2}), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin(suite) -> []; +ix_match_w_plugin(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im1, [{attributes, [k, v1, v2]}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im1, set), + ?verify_mnesia(Nodes, []). + + +ix_match_w_plugin_ordered(suite) -> []; +ix_match_w_plugin_ordered(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im2, [{attributes, [k, v1, v2]}, + {type, ordered_set}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im2, ordered_set), + ?verify_mnesia(Nodes, []). + +ix_match_w_plugin_bag(suite) -> []; +ix_match_w_plugin_bag(Config) when is_list(Config) -> + [_N1] = Nodes = ?acquire_nodes(1, Config), + ok = add_plugin(), + {atomic, ok} = mnesia:create_table(im3, [{attributes, [k, v1, v2]}, + {type, bag}, + {index, [{{pfx}, ordered}, + {v1, ordered}]}]), + fill_and_test_index_match(im3, bag), + ?verify_mnesia(Nodes, []). + +fill_and_test_index_match(Tab, Type) -> + [ok,ok,ok,ok,ok,ok,ok,ok,ok] = + [mnesia:dirty_write({Tab, K, V1, V2}) + || {K,V1,V2} <- [{1,a,"123"}, + {2,b,"12345"}, + {3,c,"123"}, + {4,d,nil}, + {5,e,nil}, + {6,f,nil}, + {7,g,nil}, %% overwritten if not bag + {7,g,"234"}, + {8,h,"123"}]], + mnesia:activity( + transaction, + fun() -> + ok = mnesia:write({Tab, 1, aa, "1234"}), %% replaces if not bag + ok = mnesia:delete({Tab, 2}), + ok = mnesia:delete({Tab, 4}), + ok = mnesia:write({Tab, 6, ff, nil}), + ok = mnesia:write({Tab, 7, gg, "123"}), + ok = mnesia:write({Tab, 100, x, nil}), + ok = mnesia:delete_object({Tab,3,c,"123"}), + ok = mnesia:delete_object({Tab,5,e,nil}), + Res = mnesia:index_read(Tab, <<"123">>, {pfx}), + SetRes = [{Tab,1,aa,"1234"}, {Tab,7,gg,"123"}, {Tab,8,h,"123"}], + case Type of + set -> + SetRes = lists:sort(Res); + ordered_set -> + SetRes = Res; + bag -> + [{Tab,1,a,"123"}, {Tab,1,aa,"1234"}, + {Tab,7,gg,"123"}, {Tab,8,h,"123"}] = lists:sort(Res) + end + end). + +%% ============================================================ +%% +ix_prefixes(_Tab, _Pos, Obj) -> + lists:foldl( + fun(V, Acc) when is_list(V) -> + try Pfxs = prefixes(list_to_binary(V)), + Pfxs ++ Acc + catch + error:_ -> + Acc + end; + (V, Acc) when is_binary(V) -> + Pfxs = prefixes(V), + Pfxs ++ Acc; + (_, Acc) -> + Acc + end, [], tl(tuple_to_list(Obj))). + +ix_prefixes2(Tab, Pos, Obj) -> + [rev(P) || P <- ix_prefixes(Tab, Pos, Obj)]. + +rev(B) when is_binary(B) -> + list_to_binary(lists:reverse(binary_to_list(B))). + +prefixes(<<P:3/binary, _/binary>>) -> + [P]; +prefixes(_) -> + []. diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 5a981bf539..037d6adb38 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -67,6 +67,7 @@ alias(recovery) -> mnesia_recovery_test; alias(registry) -> mnesia_registry_test; alias(suite) -> mnesia_SUITE; alias(trans) -> mnesia_trans_access_test; +alias(ixp) -> mnesia_index_plugin_test; alias(Other) -> Other. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/observer/test/observer_SUITE.erl b/lib/observer/test/observer_SUITE.erl index 75336cedcc..7d54bb9b3b 100644 --- a/lib/observer/test/observer_SUITE.erl +++ b/lib/observer/test/observer_SUITE.erl @@ -41,7 +41,8 @@ %% Default timetrap timeout (set in init_per_testcase) -define(default_timeout, ?t:minutes(2)). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> [{timetrap, {minutes, 5}}, + {ct_hooks,[ts_install_cth]}]. all() -> [app_file, appup_file, {group, gui}]. diff --git a/lib/public_key/asn1/OTP-PKIX.asn1 b/lib/public_key/asn1/OTP-PKIX.asn1 index 9bcd99fba3..ff3250b383 100644 --- a/lib/public_key/asn1/OTP-PKIX.asn1 +++ b/lib/public_key/asn1/OTP-PKIX.asn1 @@ -233,9 +233,13 @@ countryName ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { -- regarding how to handle and sometimes accept incorrect certificates -- we define and use the type below instead of X520countryName + -- We accept utf8String encoding of the US-ASCII + -- country name code and the mix up with other country code systems + -- that uses three characters instead of two. + OTP-X520countryname ::= CHOICE { - printableString PrintableString (SIZE (2)), - utf8String UTF8String (SIZE (2)) + printableString PrintableString (SIZE (2..3)), + utf8String UTF8String (SIZE (2..3)) } serialNumber ATTRIBUTE-TYPE-AND-VALUE-CLASS ::= { diff --git a/lib/ssl/doc/src/Makefile b/lib/ssl/doc/src/Makefile index 7cf251d8f9..064131944c 100644 --- a/lib/ssl/doc/src/Makefile +++ b/lib/ssl/doc/src/Makefile @@ -47,6 +47,7 @@ XML_CHAPTER_FILES = \ ssl_protocol.xml \ using_ssl.xml \ ssl_distribution.xml \ + standards_compliance.xml \ notes.xml BOOK_FILES = book.xml diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 37bf9033a1..74a0a0a03e 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -145,8 +145,13 @@ </datatype> <datatype> - <name name="legacy_version"/> + <name name="tls_legacy_version"/> </datatype> + + <datatype> + <name name="dtls_legacy_version"/> + </datatype> + <datatype> <name name="prf_random"/> diff --git a/lib/ssl/doc/src/ssl_app.xml b/lib/ssl/doc/src/ssl_app.xml index 893919aeb4..b05caf44ea 100644 --- a/lib/ssl/doc/src/ssl_app.xml +++ b/lib/ssl/doc/src/ssl_app.xml @@ -35,45 +35,10 @@ <description> <p> - The ssl application is an implementation of the SSL/TLS/DTLS protocol in Erlang. + The ssl application is an implementation of the SSL, TLS and DTLS protocols in Erlang. </p> - <list type="bulleted"> - <item>Supported SSL/TLS/DTLS-versions are SSL-3.0, TLS-1.0, - TLS-1.1, TLS-1.2, DTLS-1.0 (based on TLS-1.1), DTLS-1.2 (based on TLS-1.2)</item> - <item>For security reasons SSL-2.0 is not supported. - Interoperability with SSL-2.0 enabled clients dropped. (OTP 21) </item> - <item>For security reasons SSL-3.0 is no longer supported by default, - but can be configured. (OTP 19) </item> - <item>For security reasons RSA key exchange cipher suites are no longer supported by default, - but can be configured. (OTP 21) </item> - <item>For security reasons DES cipher suites are no longer supported by default, - but can be configured. (OTP 20) </item> - <item>For security reasons 3DES cipher suites are no longer supported by default, - but can be configured. (OTP 21) </item> - <item> Renegotiation Indication Extension <url href="http://www.ietf.org/rfc/rfc5746.txt">RFC 5746</url> is supported - </item> - <item>Ephemeral Diffie-Hellman cipher suites are supported, - but not Diffie Hellman Certificates cipher suites.</item> - <item>Elliptic Curve cipher suites are supported if the Crypto - application supports it and named curves are used. - </item> - <item>Export cipher suites are not supported as the - U.S. lifted its export restrictions in early 2000.</item> - <item>IDEA cipher suites are not supported as they have - become deprecated by the latest TLS specification so it is not - motivated to implement them.</item> - <item>Compression is not supported.</item> - <item>CRL validation is supported.</item> - <item>Policy certificate extensions are not supported.</item> - <item>'Server Name Indication' extension - (<url href="http://www.ietf.org/rfc/rfc6066.txt">RFC 6066</url>) is supported.</item> - <item>Application Layer Protocol Negotiation (ALPN) and its successor Next Protocol Negotiation (NPN) - are supported. </item> - <item>It is possible to use Pre-Shared Key (PSK) and Secure Remote Password (SRP) - cipher suites, but they are not enabled by default. - </item> - </list> - </description> + <p>For current statement of standards compliance see the <seealso marker="standards_compliance">User's Guide</seealso>.</p> + </description> <section> <title>DEPENDENCIES</title> diff --git a/lib/ssl/doc/src/standards_compliance.xml b/lib/ssl/doc/src/standards_compliance.xml new file mode 100644 index 0000000000..c20bab4e50 --- /dev/null +++ b/lib/ssl/doc/src/standards_compliance.xml @@ -0,0 +1,2312 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2015</year> + <year>2019</year> + <holder>Ericsson AB, All Rights Reserved</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Initial Developer of the Original Code is Ericsson AB. + </legalnotice> + + <title>Standards Compliance</title> + <prepared>OTP team</prepared> + <docno></docno> + <date>2019-03-20</date> + <rev>A</rev> + <file>standards_compliance.xml</file> + </header> + + <section> + <title>Purpose</title> + <p>This section describes the current state of standards compliance of the ssl application.</p> + </section> + + <section> + <title>Common (pre TLS 1.3)</title> + <list type="bulleted"> + <item>For security reasons RSA key exchange cipher suites are no longer supported by default, + but can be configured. (OTP 21) + </item> + <item>For security reasons DES cipher suites are no longer supported by default, + but can be configured. (OTP 20) + </item> + <item>For security reasons 3DES cipher suites are no longer supported by default, + but can be configured. (OTP 21) + </item> + <item>Renegotiation Indication Extension <url href="http://www.ietf.org/rfc/rfc5746.txt">RFC 5746</url> is supported + </item> + <item>Ephemeral Diffie-Hellman cipher suites are supported, + but not Diffie Hellman Certificates cipher suites. + </item> + <item>Elliptic Curve cipher suites are supported if the Crypto + application supports it and named curves are used. + </item> + <item>Export cipher suites are not supported as the + U.S. lifted its export restrictions in early 2000. + </item> + <item>IDEA cipher suites are not supported as they have + become deprecated by the TLS 1.2 specification so it is not + motivated to implement them. + </item> + <item>Compression is not supported. + </item> + </list> + </section> + + <section> + <title>Common</title> + <list type="bulleted"> + <item>CRL validation is supported.</item> + <item>Policy certificate extensions are not supported.</item> + <item>'Server Name Indication' extension + (<url href="http://www.ietf.org/rfc/rfc6066.txt">RFC 6066</url>) is supported.</item> + <item>Application Layer Protocol Negotiation (ALPN) and its successor Next Protocol Negotiation (NPN) are supported. </item> + <item>It is possible to use Pre-Shared Key (PSK) and Secure Remote Password (SRP) + cipher suites, but they are not enabled by default. + </item> + </list> + </section> + + + <section> + <title>SSL 2.0</title> + <p>For security reasons SSL-2.0 is not supported. Interoperability with SSL-2.0 enabled clients dropped. (OTP 21)</p> + </section> + + <section> + <title>SSL 3.0</title> + <p>For security reasons SSL-3.0 is no longer supported by default, but can be configured. (OTP 19)</p> + </section> + + <section> + <title>TLS 1.0</title> + <p>For security reasons TLS-1.0 is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>TLS 1.1</title> + <p>For security reasons TLS-1.1 is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>TLS 1.2</title> + <p>Supported</p> + </section> + + <section> + <title>DTLS 1.0</title> + <p>For security reasons DTLS-1.0 (based on TLS 1.1) is no longer supported by default, but can be configured. (OTP 22)</p> + </section> + + <section> + <title>DTLS 1.2</title> + <p>Supported (based on TLS 1.2)</p> + </section> + + <section> + <title>DTLS 1.3</title> + <p>Not yet supported</p> + </section> + + <section> + <title>TLS 1.3</title> + <p> This section describes the current state of standards compliance for TLS 1.3.</p> + <p>(C = Compliant, NC = Non-Compliant, P = Partially-Compliant, NA = Not Applicable)</p> + <table> + <row> + <cell align="left" valign="middle"><em>Section</em></cell> + <cell align="left" valign="middle"><em>Feature</em></cell> + <cell align="left" valign="middle"><em>State</em></cell> + <cell align="left" valign="middle"><em>Since</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-1.2"> + 1.3. Updates Affecting TLS 1.2 + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection mechanism</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">RSASSA-PSS signature schemes</cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (ClientHello) extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2"> + 2. Protocol Overview + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">(EC)DHE</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">PSK-only</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">PSK with (EC)DHE</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.1"> + 2.1. Incorrect DHE share + </url> + </cell> + <cell align="left" valign="middle">HelloRetryRequest</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.2"> + 2.2. Resumption and Pre-Shared Key (PSK) + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-2.3"> + 2.3. 0-RTT Data + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.1"> + 4.1.1. Cryptographic Negotiation + </url> + </cell> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms extension</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key extension</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.2"> + 4.1.2. Client Hello + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">padding (RFC7685)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">psk_key_exchange_modes (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446) </cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">post_handshake_auth (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">padding (RFC7685)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">psk_key_exchange_modes (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446) </cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">post_handshake_auth (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.3"> + 4.1.3. Server Hello + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Version downgrade protection</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">pre_shared_key (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.1.4"> + 4.1.4. Hello Retry Request + </url> + </cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">key_share (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">cookie (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.1"> + 4.2.1. Supported Versions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.2"> + 4.2.2. Cookie + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.3"> + 4.2.3. Signature Algorithms + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_sha1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha512</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp384r1_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_secp521r1_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_rsae_sha512</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ed448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha384</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pss_pss_sha512</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">rsa_pkcs1_sha1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ecdsa_sha1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.4"> + 4.2.4. Certificate Authorities + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.5"> + 4.2.5. OID Filters + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.6"> + 4.2.6. Post-Handshake Client Authentication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.7"> + 4.2.7. Supported Groups + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp256r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp384r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp521r1</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x25519</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x448</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe2048</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe3072</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe4096</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe6144</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe8192</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp256r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp384r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">secp521r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x25519</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">x448</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe2048</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe3072</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe4096</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe6144</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">ffdhe8192</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.8"> + 4.2.8. Key Share + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.9"> + 4.2.9. Pre-Shared Key Exchange Modes + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.10"> + 4.2.10. Early Data Indication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11"> + 4.2.11. Pre-Shared Key Extension + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.1"> + 4.2.11.1. Ticket Age + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.2"> + 4.2.11.2. PSK Binder + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.2.11.3"> + 4.2.11.3. Processing Order + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.3.1"> + 4.3.1. Encrypted Extensions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">max_fragment_length (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_groups (RFC7919)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">use_srtp (RFC5764)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">heartbeat (RFC6520)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">application_layer_protocol_negotiation (RFC7301)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">client_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_certificate_type (RFC7250)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">supported_versions (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.3.2"> + 4.3.2. Certificate Request + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate_authorities (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">oid_filters (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signature_algorithms_cert (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.1"> + 4.4.1. The Transcript Hash + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2"> + 4.4.2. Certificate + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">status_request (RFC6066)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">signed_certificate_timestamp (RFC6962)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.1"> + 4.4.2.1. OCSP Status and SCT Extensions + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.2"> + 4.4.2.2. Server Certificate Selection + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate type MUST be X.509v3</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate's public key is compatible</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The certificate MUST allow the key to be used for signing</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name and certificate_authorities are used</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate type MUST be X.509v3</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">certificate's public key is compatible</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The certificate MUST allow the key to be used for signing</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">server_name and certificate_authorities are used</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.3"> + 4.4.2.3. Client Certificate Selection + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.2.4"> + 4.4.2.4. Receiving a Certificate Message + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.3"> + 4.4.3. Certificate Verify + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.4.4"> + 4.4.4. Finished + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.5"> + 4.5. End of Early Data + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.1"> + 4.6.1. New Session Ticket Message + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">early_data (RFC8446)</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.2"> + 4.6.2. Post-Handshake Authentication + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-4.6.3"> + 4.6.3. Key and Initialization Vector Update + </url> + </cell> + <cell align="left" valign="middle"><em>Client</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Server</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.1"> + 5.1. Record Layer + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT be interleaved with other record types</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT span key changes</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT send zero-length fragments</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Alert messages MUST NOT be fragmented</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.2"> + 5.2. Record Payload Protection + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.3"> + 5.3. Per-Record Nonce + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.4"> + 5.4. Record Padding + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MAY choose to pad</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST NOT send Handshake and Alert records that have a zero-length TLSInnerPlaintext.content</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">The padding sent is automatically verified</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-5.5"> + 5.5. Limits on Key Usage + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-6.1"> + 6.1. Closure Alerts + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">close_notify</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">user_cancelled</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-6.2"> + 6.2. Error Alerts + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.1"> + 7.1. Key Schedule + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.2"> + 7.2. Updating Traffic Secrets + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.3"> + 7.3. Traffic Key Calculation + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-7.5"> + 7.5. Exporters + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8"> + 8. 0-RTT and Anti-Replay + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.1"> + 8.1. Single-Use Tickets + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.2"> + 8.2. Client Hello Recording + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-8.3"> + 8.3. Freshness Checks + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.1"> + 9.1. Mandatory-to-Implement Cipher Suites + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST implement the TLS_AES_128_GCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD implement the TLS_AES_256_GCM_SHA384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD implement the TLS_CHACHA20_POLY1305_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Digital signatures</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support rsa_pkcs1_sha256 (for certificates)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support rsa_pss_rsae_sha256 (for CertificateVerify and certificates)</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support ecdsa_secp256r1_sha256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>Key Exchange</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support key exchange with secp256r1</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">SHOULD support key exchange with X25519</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.2"> + 9.2. Mandatory-to-Implement Extensions + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Supported Versions</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Cookie</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Signature Algorithms</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Signature Algorithms Certificate</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Negotiated Groups</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Key Share</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">Server Name Indication</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>MUST send and use these extensions</em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"supported_versions" is REQUIRED for ClientHello, ServerHello and HelloRetryRequest</cell> + <cell align="left" valign="middle"><em>PC</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"signature_algorithms" is REQUIRED for certificate authentication</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"supported_groups" is REQUIRED for ClientHello messages using (EC)DHE key exchange</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"key_share" is REQUIRED for (EC)DHE key exchange</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"pre_shared_key" is REQUIRED for PSK key agreement</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">"psk_key_exchange_modes" is REQUIRED for PSK key agreement</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>TLS 1.3 ClientHello</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">If not containing a "pre_shared_key" extension, it MUST contain both a "signature_algorithms" extension and a "supported_groups" extension.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">If containing a "supported_groups" extension, it MUST also contain a "key_share" extension, and vice versa. An empty KeyShare.client_shares vector is permitted.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>TLS 1.3 ServerHello</em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">MUST support the use of the "server_name" extension</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-9.3"> + 9.3. Protocol Invariants + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle"><em>MUST correctly handle extensible fields</em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A client sending a ClientHello MUST support all parameters advertised in it.</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A middlebox which terminates a TLS connection MUST behave as a compliant TLS server</cell> + <cell align="left" valign="middle"><em>NA</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">A middlebox which forwards ClientHello parameters it does not understand MUST NOT process any messages beyond that ClientHello.</cell> + <cell align="left" valign="middle"><em>NA</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-B.4"> + B.4. Cipher Suites + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_GCM_SHA256</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_256_GCM_SHA384</cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle">22</cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_CHACHA20_POLY1305_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_CCM_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + <row> + <cell align="left" valign="middle"></cell> + <cell align="left" valign="middle">TLS_AES_128_CCM_8_SHA256</cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.1"> + C.1. Random Number Generation and Seeding + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.2"> + C.2. Certificates and Authentication + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.3"> + C.3. Implementation Pitfalls + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.4"> + C.4. Client Tracking Prevention + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-C.5"> + C.5. Unauthenticated Operation + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.1"> + D.1. Negotiating with an Older Server + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.2"> + D.2. Negotiating with an Older Client + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.3"> + D.3. 0-RTT Backward Compatibility + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>NC</em></cell> + <cell align="left" valign="middle"><em></em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.4"> + D.4. Middlebox Compatibility Mode + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>P</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <row> + <cell align="left" valign="middle"> + <url href="https://tools.ietf.org/html/rfc8446#section-D.5"> + D.5. Security Restrictions Related to Backward Compatibility + </url> + </cell> + <cell align="left" valign="middle"><em></em></cell> + <cell align="left" valign="middle"><em>C</em></cell> + <cell align="left" valign="middle"><em>22</em></cell> + </row> + + <tcaption>Standards Compliance</tcaption> + </table> + + </section> + +</chapter> diff --git a/lib/ssl/doc/src/usersguide.xml b/lib/ssl/doc/src/usersguide.xml index 23ccf668c3..b22b2456e4 100644 --- a/lib/ssl/doc/src/usersguide.xml +++ b/lib/ssl/doc/src/usersguide.xml @@ -38,6 +38,7 @@ <xi:include href="ssl_protocol.xml"/> <xi:include href="using_ssl.xml"/> <xi:include href="ssl_distribution.xml"/> + <xi:include href="standards_compliance.xml"/> </part> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index ed47980a69..7993be8a74 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -193,7 +193,8 @@ next_event(StateName, no_record, %% TODO maybe buffer later epoch next_event(StateName, no_record, State, Actions); {#alert{} = Alert, State} -> - {next_state, StateName, State, [{next_event, internal, Alert} | Actions]} + Version = State#state.connection_env#connection_env.negotiated_version, + handle_own_alert(Alert, Version, StateName, State) end; next_event(connection = StateName, Record, #state{connection_states = #{current_read := #{epoch := CurrentEpoch}}} = State0, Actions) -> @@ -233,7 +234,8 @@ next_event(StateName, Record, %% TODO maybe buffer later epoch next_event(StateName, no_record, State0, Actions); #alert{} = Alert -> - {next_state, StateName, State0, [{next_event, internal, Alert} | Actions]} + Version = State0#state.connection_env#connection_env.negotiated_version, + handle_own_alert(Alert, Version, StateName, State0) end. %%% DTLS record protocol level application data messages @@ -840,7 +842,7 @@ next_dtls_record(Data, StateName, #state{protocol_buffers = #protocol_buffers{ end. acceptable_record_versions(hello, _) -> - [dtls_record:protocol_version(Vsn) || Vsn <- ?ALL_DATAGRAM_SUPPORTED_VERSIONS]; + [dtls_record:protocol_version(Vsn) || Vsn <- ?ALL_AVAILABLE_DATAGRAM_VERSIONS]; acceptable_record_versions(_, #state{connection_env = #connection_env{negotiated_version = Version}}) -> [Version]. @@ -1075,10 +1077,10 @@ start_retransmision_timer(Timeout, #state{protocol_specific = PS} = State) -> {State#state{protocol_specific = PS#{flight_state => {retransmit, new_timeout(Timeout)}}}, [{state_timeout, Timeout, flight_retransmission_timeout}]}. -new_timeout(N) when N =< 30 -> +new_timeout(N) when N =< 30000 -> N * 2; new_timeout(_) -> - 60. + 60000. send_handshake_flight(#state{static_env = #static_env{socket = Socket, transport_cb = Transport}, diff --git a/lib/ssl/src/dtls_packet_demux.erl b/lib/ssl/src/dtls_packet_demux.erl index e0423b07b4..2e9184b7ac 100644 --- a/lib/ssl/src/dtls_packet_demux.erl +++ b/lib/ssl/src/dtls_packet_demux.erl @@ -203,9 +203,9 @@ dispatch(Client, Msg, #state{dtls_msq_queues = MsgQueues} = State) -> Pid ! Msg, State#state{dtls_msq_queues = kv_update(Client, Queue, MsgQueues)}; - {{value, _}, Queue} -> + {{value, _UDP}, _Queue} -> State#state{dtls_msq_queues = - kv_update(Client, queue:in(Msg, Queue), MsgQueues)}; + kv_update(Client, queue:in(Msg, Queue0), MsgQueues)}; {empty, Queue} -> State#state{dtls_msq_queues = kv_update(Client, queue:in(Msg, Queue), MsgQueues)} diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index e7fab7ebc5..8d9b92361b 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011-2018. All Rights Reserved. +%% Copyright Ericsson AB 2011-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -132,8 +132,8 @@ f_recv(SslSocket, Length, Timeout) -> f_setopts_pre_nodeup(_SslSocket) -> ok. -f_setopts_post_nodeup(_SslSocket) -> - ok. +f_setopts_post_nodeup(SslSocket) -> + ssl:setopts(SslSocket, [nodelay()]). f_getll(DistCtrl) -> {ok, DistCtrl}. @@ -199,7 +199,7 @@ listen(Name) -> gen_listen(Driver, Name) -> case inet_tcp_dist:gen_listen(Driver, Name) of {ok, {Socket, Address, Creation}} -> - inet:setopts(Socket, [{packet, 4}]), + inet:setopts(Socket, [{packet, 4}, {nodelay, true}]), {ok, {Socket, Address#net_address{protocol=tls}, Creation}}; Other -> Other @@ -532,7 +532,7 @@ do_setup_connect(Driver, Kernel, Node, Address, Ip, TcpPort, Version, Type, MyNo case ssl:connect( Address, TcpPort, [binary, {active, false}, {packet, 4}, - Driver:family(), nodelay()] ++ Opts, + Driver:family(), {nodelay, true}] ++ Opts, net_kernel:connecttime()) of {ok, #sslsocket{pid = [_, DistCtrl| _]} = SslSocket} -> _ = monitor_pid(DistCtrl), diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index bfa349c8d8..8807c575b1 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -103,14 +103,19 @@ -type ip_address() :: inet:ip_address(). -type session_id() :: binary(). -type protocol_version() :: tls_version() | dtls_version(). --type tls_version() :: tlsv1 | 'tlsv1.1' | 'tlsv1.2' | 'tlsv1.3' | legacy_version(). --type dtls_version() :: 'dtlsv1' | 'dtlsv1.2'. --type legacy_version() :: sslv3. +-type tls_version() :: 'tlsv1.2' | 'tlsv1.3' | tls_legacy_version(). +-type dtls_version() :: 'dtlsv1.2' | dtls_legacy_version(). +-type tls_legacy_version() :: tlsv1 | 'tlsv1.1' | sslv3. +-type dtls_legacy_version() :: 'dtlsv1'. -type verify_type() :: verify_none | verify_peer. -type cipher() :: aes_128_cbc | aes_256_cbc | aes_128_gcm | aes_256_gcm | + aes_128_ccm | + aes_256_ccm | + aes_128_ccm_8 | + aes_256_ccm_8 | chacha20_poly1305 | legacy_cipher(). -type legacy_cipher() :: rc4_128 | diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index fe8736d2df..97878431a6 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -35,7 +35,7 @@ -include_lib("public_key/include/public_key.hrl"). -export([security_parameters/2, security_parameters/3, security_parameters_1_3/2, - cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/5, aead_decrypt/6, + cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/6, aead_decrypt/6, suites/1, all_suites/1, crypto_support_filters/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, psk_suites_anon/1, srp_suites/0, srp_suites_anon/0, @@ -106,9 +106,13 @@ security_parameters_1_3(SecParams, CipherSuite) -> cipher_init(?RC4, IV, Key) -> State = crypto:stream_init(rc4, Key), #cipher_state{iv = IV, key = Key, state = State}; -cipher_init(?AES_GCM, IV, Key) -> +cipher_init(Type, IV, Key) when Type == ?AES_GCM; + Type == ?AES_CCM -> <<Nonce:64>> = random_bytes(8), #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 16}; +cipher_init(?AES_CCM_8, IV, Key) -> + <<Nonce:64>> = random_bytes(8), + #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 8}; cipher_init(?CHACHA20_POLY1305, IV, Key) -> #cipher_state{iv = IV, key = Key, tag_len = 16}; cipher_init(_BCA, IV, Key) -> @@ -148,14 +152,18 @@ cipher(?AES_CBC, CipherState, Mac, Fragment, Version) -> crypto:block_encrypt(aes_cbc256, Key, IV, T) end, block_size(aes_128_cbc), CipherState, Mac, Fragment, Version). -aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData) -> - crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment}). +aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData, TagLen) -> + crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment, TagLen}). aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AdditionalData) -> crypto:block_decrypt(aead_type(Type), Key, Nonce, {AdditionalData, CipherText, CipherTag}). aead_type(?AES_GCM) -> aes_gcm; +aead_type(?AES_CCM) -> + aes_ccm; +aead_type(?AES_CCM_8) -> + aes_ccm; aead_type(?CHACHA20_POLY1305) -> chacha20_poly1305. @@ -311,8 +319,7 @@ anonymous_suites({254, _} = Version) -> dtls_v1:anonymous_suites(Version); anonymous_suites(4) -> []; %% Raw public key negotiation may be used instead -anonymous_suites(N) - when N >= 3 -> +anonymous_suites( 3 = N) -> psk_suites_anon(N) ++ [?TLS_DH_anon_WITH_AES_128_GCM_SHA256, ?TLS_DH_anon_WITH_AES_256_GCM_SHA384, @@ -347,8 +354,7 @@ psk_suites({3, N}) -> psk_suites(N); psk_suites(4) -> []; %% TODO Add new PSK, PSK_(EC)DHE suites -psk_suites(N) - when N >= 3 -> +psk_suites(3) -> [ ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, @@ -369,20 +375,32 @@ psk_suites(_) -> %%-------------------------------------------------------------------- psk_suites_anon({3, N}) -> psk_suites_anon(N); -psk_suites_anon(N) - when N >= 3 -> +psk_suites_anon(3) -> [ ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_PSK_WITH_AES_256_GCM_SHA384, ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_PSK_WITH_AES_256_CBC_SHA384, + ?TLS_DHE_PSK_WITH_AES_256_CCM, + ?TLS_PSK_DHE_WITH_AES_256_CCM_8, + ?TLS_PSK_WITH_AES_256_CCM, + ?TLS_PSK_WITH_AES_256_CCM_8, ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, - ?TLS_PSK_WITH_AES_128_CBC_SHA256 + ?TLS_PSK_WITH_AES_128_CBC_SHA256, + ?TLS_DHE_PSK_WITH_AES_128_CCM, + ?TLS_PSK_DHE_WITH_AES_128_CCM_8, + ?TLS_PSK_WITH_AES_128_CCM, + ?TLS_PSK_WITH_AES_128_CCM_8, + ?TLS_ECDHE_PSK_WITH_RC4_128_SHA ] ++ psk_suites_anon(0); psk_suites_anon(_) -> [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, @@ -589,7 +607,7 @@ is_acceptable_keyexchange(dhe_rsa, Algos) -> proplists:get_bool(dh, Algos) andalso proplists:get_bool(rsa, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_anon; - KeyExchange == ecdhe_psk -> + KeyExchange == ecdhe_psk -> proplists:get_bool(ecdh, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_ecdsa; KeyExchange == ecdhe_ecdsa -> @@ -629,6 +647,12 @@ is_acceptable_cipher(Cipher, Algos) when Cipher == aes_128_gcm; Cipher == aes_256_gcm -> proplists:get_bool(aes_gcm, Algos); +is_acceptable_cipher(Cipher, Algos) + when Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> + proplists:get_bool(aes_ccm, Algos); is_acceptable_cipher(Cipher, Algos) -> proplists:get_bool(Cipher, Algos). @@ -721,6 +745,12 @@ bulk_cipher_algorithm(Cipher) when Cipher == aes_128_cbc; bulk_cipher_algorithm(Cipher) when Cipher == aes_128_gcm; Cipher == aes_256_gcm -> ?AES_GCM; +bulk_cipher_algorithm(Cipher) when Cipher == aes_128_ccm; + Cipher == aes_256_ccm -> + ?AES_CCM; +bulk_cipher_algorithm(Cipher) when Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> + ?AES_CCM_8; bulk_cipher_algorithm(chacha20_poly1305) -> ?CHACHA20_POLY1305. @@ -735,6 +765,10 @@ type(Cipher) when Cipher == des_cbc; ?BLOCK; type(Cipher) when Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> ?AEAD. @@ -752,8 +786,16 @@ key_material(aes_256_cbc) -> 32; key_material(aes_128_gcm) -> 16; +key_material(aes_128_ccm) -> + 16; +key_material(aes_128_ccm_8) -> + 16; key_material(aes_256_gcm) -> 32; +key_material(aes_256_ccm_8) -> + 32; +key_material(aes_256_ccm) -> + 32; key_material(chacha20_poly1305) -> 32. @@ -769,6 +811,10 @@ expanded_key_material(Cipher) when Cipher == aes_128_cbc; Cipher == aes_256_cbc; Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> unknown. @@ -778,12 +824,16 @@ effective_key_bits(des_cbc) -> 56; effective_key_bits(Cipher) when Cipher == rc4_128; Cipher == aes_128_cbc; - Cipher == aes_128_gcm -> + Cipher == aes_128_gcm; + Cipher == aes_128_ccm; + Cipher == aes_128_ccm_8 -> 128; effective_key_bits('3des_ede_cbc') -> 168; effective_key_bits(Cipher) when Cipher == aes_256_cbc; Cipher == aes_256_gcm; + Cipher == aes_256_ccm; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> 256. @@ -792,7 +842,11 @@ iv_size(Cipher) when Cipher == null; Cipher == chacha20_poly1305-> 0; iv_size(Cipher) when Cipher == aes_128_gcm; - Cipher == aes_256_gcm -> + Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8 -> 4; iv_size(Cipher) -> block_size(Cipher). @@ -804,6 +858,10 @@ block_size(Cipher) when Cipher == aes_128_cbc; Cipher == aes_256_cbc; Cipher == aes_128_gcm; Cipher == aes_256_gcm; + Cipher == aes_128_ccm; + Cipher == aes_256_ccm; + Cipher == aes_128_ccm_8; + Cipher == aes_256_ccm_8; Cipher == chacha20_poly1305 -> 16. diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 00822ad9de..5d2f5e2951 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -612,6 +612,58 @@ %% TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = {0xcc, 0x15} -define(TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256, <<?BYTE(16#CC), ?BYTE(16#15)>>). + +%% RFC 6655 - TLS-1.2 cipher suites + +%% TLS_RSA_WITH_AES_128_CCM = {0xC0,0x9C} +-define(TLS_RSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#9C)>>). + +%% TLS_RSA_WITH_AES_256_CCM = {0xC0,0x9D} +-define(TLS_RSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#9D)>>). + +%% TLS_DHE_RSA_WITH_AES_256_CCM = {0xC0,0x9E} +-define(TLS_DHE_RSA_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#9E)>>). + +%% TLS_DHE_RSA_WITH_AES_128_CCM = {0xC0,0x9F} +-define(TLS_DHE_RSA_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#9F)>>). + +%% TLS_RSA_WITH_AES_256_CCM_8 = {0xC0,0x9A0} +-define(TLS_RSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A0)>>). + +%% TLS_RSA_WITH_AES_128_CCM_8 = {0xC0,0xA1} +-define(TLS_RSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A1)>>). + +%% TLS_DHE_RSA_WITH_AES_128_CCM_8 = {0xC0,0xA2} +-define(TLS_DHE_RSA_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A2)>>). + +%% TLS_DHE_RSA_WITH_AES_256_CCM_8 = {0xC0,0xA3} +-define(TLS_DHE_RSA_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A3)>>). + +%% TLS_PSK_WITH_AES_128_CCM = {0xC0,0xA4} +-define(TLS_PSK_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#A4)>>). + +%% TLS_PSK_WITH_AES_256_CCM = {0xC0,0xA5) +-define(TLS_PSK_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#A5)>>). + +%% TLS_DHE_PSK_WITH_AES_128_CCM = {0xC0,0xA6} +-define(TLS_DHE_PSK_WITH_AES_128_CCM, <<?BYTE(16#C0), ?BYTE(16#A6)>>). + +%% TLS_DHE_PSK_WITH_AES_256_CCM = {0xC0,0xA7} +-define(TLS_DHE_PSK_WITH_AES_256_CCM, <<?BYTE(16#C0), ?BYTE(16#A7)>>). + +%% TLS_PSK_WITH_AES_128_CCM_8 = {0xC0,0xA8} +-define(TLS_PSK_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A8)>>). + +%% TLS_PSK_WITH_AES_256_CCM_8 = {0xC0,0xA9) +-define(TLS_PSK_WITH_AES_256_CCM_8, <<?BYTE(16#C0), ?BYTE(16#A9)>>). + +%% TLS_PSK_DHE_WITH_AES_128_CCM_8 = {0xC0,0xAA} +-define(TLS_PSK_DHE_WITH_AES_128_CCM_8, <<?BYTE(16#C0), ?BYTE(16#AA)>>). + +%% TLS_PSK_DHE_WITH_AES_256_CCM_8 = << ?BYTE(0xC0,0xAB} +-define(TLS_PSK_DHE_WITH_AES_256_CCM_8, <<?BYTE(16#C0),?BYTE(16#AB)>>). + + %%% TLS 1.3 cipher suites RFC8446 %% TLS_AES_128_GCM_SHA256 = {0x13,0x01} diff --git a/lib/ssl/src/ssl_cipher_format.erl b/lib/ssl/src/ssl_cipher_format.erl index b592295d56..8737181922 100644 --- a/lib/ssl/src/ssl_cipher_format.erl +++ b/lib/ssl/src/ssl_cipher_format.erl @@ -467,16 +467,16 @@ suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> cipher => aes_256_gcm, mac => null, prf => sha384}; -%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> -%% #{key_exchange => ecdhe_psk, -%% cipher => aes_128_ccm, -%% mac => null, -%% prf =>sha256}; -%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> -%% #{key_exchange => ecdhe_psk, -%% cipher => aes_256_ccm, -%% mac => null, -%% prf => sha256}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> + #{key_exchange => ecdhe_psk, + cipher => aes_128_ccm, + mac => null, + prf =>sha256}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> + #{key_exchange => ecdhe_psk, + cipher => aes_128_ccm_8, + mac => null, + prf =>sha256}; %%% SRP Cipher Suites RFC 5054 suite_definition(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> #{key_exchange => srp_anon, @@ -792,7 +792,53 @@ suite_definition(?TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384) -> cipher => aes_256_gcm, mac => aead, prf => sha384}; -%% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites +suite_definition(?TLS_PSK_WITH_AES_128_CCM) -> + #{key_exchange => psk, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_WITH_AES_256_CCM) -> + #{key_exchange => psk, + cipher => aes_256_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_DHE_PSK_WITH_AES_128_CCM) -> + #{key_exchange => dhe_psk, + cipher => aes_128_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_DHE_PSK_WITH_AES_256_CCM) -> + #{key_exchange => dhe_psk, + cipher => aes_256_ccm, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_WITH_AES_128_CCM_8) -> + #{key_exchange => psk, + cipher => aes_128_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_WITH_AES_256_CCM_8) -> + #{key_exchange => psk, + cipher => aes_256_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_DHE_WITH_AES_128_CCM_8) -> + #{key_exchange => dhe_psk, + cipher => aes_128_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(?TLS_PSK_DHE_WITH_AES_256_CCM_8) -> + #{key_exchange => dhe_psk, + cipher => aes_256_ccm_8, + mac => aead, + prf => sha256}; +suite_definition(#{key_exchange := psk_dhe, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_256_CCM_8; + +% draft-agl-tls-chacha20poly1305-04 Chacha20/Poly1305 Suites suite_definition(?TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) -> #{key_exchange => ecdhe_rsa, cipher => chacha20_poly1305, @@ -825,16 +871,15 @@ suite_definition(?TLS_CHACHA20_POLY1305_SHA256) -> mac => aead, prf => sha256}. %% suite_definition(?TLS_AES_128_CCM_SHA256) -> -%% #{key_exchange => any, -%% cipher => aes_128_ccm, -%% mac => aead, -%% prf => sha256}; +%% #{key_exchange => any, +%% cipher => aes_128_ccm, +%% mac => aead, +%% prf => sha256}; %% suite_definition(?TLS_AES_128_CCM_8_SHA256) -> -%% #{key_exchange => any, +%% #{key_exchange => any, %% cipher => aes_128_ccm_8, -%% mac => aead, -%% prf => sha256}. - +%% mac => aead, +%% prf => sha256}. %%-------------------------------------------------------------------- -spec erl_suite_definition(cipher_suite() | internal_erl_cipher_suite()) -> old_erl_cipher_suite(). @@ -1154,16 +1199,16 @@ suite(#{key_exchange := ecdhe_psk, mac := null, prf := sha384}) -> ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384; - %% suite(#{key_exchange := ecdhe_psk, - %% cipher := aes_128_ccm, - %% mac := null, - %% prf := sha256}) -> - %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; - %% suite(#{key_exchange := ecdhe_psk, - %% cipher := aes_256_ccm, - %% mac := null, - %% prf := sha256}) -> - %% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_ccm_8, + mac := null, + prf := sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; +suite(#{key_exchange := ecdhe_psk, + cipher := aes_128_ccm, + mac := null, + prf := sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; %%% SRP Cipher Suites RFC 5054 suite(#{key_exchange := srp_anon, cipher := '3des_ede_cbc', @@ -1460,6 +1505,90 @@ suite(#{key_exchange := dhe_rsa, mac := aead, prf := sha256}) -> ?TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256; + +%% RFC 6655 - TLS-1.2 cipher suites +suite(#{key_exchange := psk, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_128_CCM; +suite(#{key_exchange := psk, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_256_CCM; +suite(#{key_exchange := dhe_psk, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_PSK_WITH_AES_128_CCM; +suite(#{key_exchange := dhe_psk, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_PSK_WITH_AES_256_CCM; +suite(#{key_exchange := rsa, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_128_CCM; +suite(#{key_exchange := rsa, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_256_CCM; +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_128_CCM; +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_ccm, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_256_CCM; + +suite(#{key_exchange := psk, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_128_CCM_8; +suite(#{key_exchange := psk, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_WITH_AES_256_CCM_8; +suite(#{key_exchange := dhe_psk, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_128_CCM_8; +suite(#{key_exchange := dhe_psk, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_PSK_DHE_WITH_AES_256_CCM_8; +suite(#{key_exchange := rsa, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_128_CCM_8; +suite(#{key_exchange := rsa, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_RSA_WITH_AES_256_CCM_8; +suite(#{key_exchange := dhe_rsa, + cipher := aes_128_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_128_CCM_8; +suite(#{key_exchange := dhe_rsa, + cipher := aes_256_ccm_8, + mac := aead, + prf := sha256}) -> + ?TLS_DHE_RSA_WITH_AES_256_CCM_8; + %% TLS 1.3 Cipher Suites RFC8446 suite(#{key_exchange := any, cipher := aes_128_gcm, diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 3d117a655f..4ee0230d88 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -72,12 +72,13 @@ %% sslv3 is considered insecure due to lack of padding check (Poodle attack) %% Keep as interop with legacy software but do not support as default +%% tlsv1.0 and tlsv1.1 is now also considered legacy %% tlsv1.3 is under development (experimental). -define(ALL_AVAILABLE_VERSIONS, ['tlsv1.3', 'tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). -define(ALL_AVAILABLE_DATAGRAM_VERSIONS, ['dtlsv1.2', dtlsv1]). %% Defines the default versions when not specified by an ssl option. --define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1]). --define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1', tlsv1]). +-define(ALL_SUPPORTED_VERSIONS, ['tlsv1.2']). +-define(MIN_SUPPORTED_VERSIONS, ['tlsv1.1']). %% Versions allowed in TLSCiphertext.version (TLS 1.2 and prior) and %% TLSCiphertext.legacy_record_version (TLS 1.3). @@ -86,7 +87,7 @@ %% Thus, the allowed range is limited to 0x0300 - 0x0303. -define(ALL_TLS_RECORD_VERSIONS, ['tlsv1.2', 'tlsv1.1', tlsv1, sslv3]). --define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2', dtlsv1]). +-define(ALL_DATAGRAM_SUPPORTED_VERSIONS, ['dtlsv1.2']). -define(MIN_DATAGRAM_SUPPORTED_VERSIONS, [dtlsv1]). %% TLS 1.3 - Section 4.1.3 diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 91f1876980..9cc131c3cb 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -471,34 +471,41 @@ initial_security_params(ConnectionEnd) -> -define(end_additional_data(AAD, Len), << (begin(AAD)end)/binary, ?UINT16(begin(Len)end) >>). -do_cipher_aead(?CHACHA20_POLY1305 = Type, Fragment, #cipher_state{key=Key} = CipherState, AAD0) -> +do_cipher_aead(?CHACHA20_POLY1305 = Type, Fragment, #cipher_state{key=Key, tag_len = TagLen} = CipherState, AAD0) -> AAD = ?end_additional_data(AAD0, erlang:iolist_size(Fragment)), Nonce = encrypt_nonce(Type, CipherState), - {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD, TagLen), {<<Content/binary, CipherTag/binary>>, CipherState}; -do_cipher_aead(Type, Fragment, #cipher_state{key=Key, nonce = ExplicitNonce} = CipherState, AAD0) -> +do_cipher_aead(Type, Fragment, #cipher_state{key=Key, tag_len = TagLen, nonce = ExplicitNonce} = CipherState, AAD0) -> AAD = ?end_additional_data(AAD0, erlang:iolist_size(Fragment)), Nonce = encrypt_nonce(Type, CipherState), - {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD, TagLen), {<<ExplicitNonce:64/integer, Content/binary, CipherTag/binary>>, CipherState#cipher_state{nonce = ExplicitNonce + 1}}. encrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}) -> crypto:exor(<<?UINT32(0), Nonce/binary>>, IV); -encrypt_nonce(?AES_GCM, #cipher_state{iv = IV, nonce = ExplicitNonce}) -> +encrypt_nonce(Type, #cipher_state{iv = IV, nonce = ExplicitNonce}) when Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> <<Salt:4/bytes, _/binary>> = IV, <<Salt/binary, ExplicitNonce:64/integer>>. decrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}, _) -> crypto:exor(<<Nonce:96/unsigned-big-integer>>, IV); -decrypt_nonce(?AES_GCM, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) -> - <<Salt/binary, ExplicitNonce/binary>>. +decrypt_nonce(Type, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) when + Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> + <<Salt/binary, ExplicitNonce/binary>>. -compile({inline, [aead_ciphertext_split/4]}). aead_ciphertext_split(?CHACHA20_POLY1305, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> CipherLen = byte_size(CipherTextFragment) - Len, <<CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, {?end_additional_data(AAD, CipherLen), CipherText, CipherTag}; -aead_ciphertext_split(?AES_GCM, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> +aead_ciphertext_split(Type, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) when Type == ?AES_GCM; + Type == ?AES_CCM; + Type == ?AES_CCM_8 -> CipherLen = byte_size(CipherTextFragment) - (Len + 8), %% 8 is length of explicit Nonce << _:8/bytes, CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, {?end_additional_data(AAD, CipherLen), CipherText, CipherTag}. diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index eb718fd20c..6d4d47cedb 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -96,6 +96,11 @@ -define(AES_CBC, 7). -define(AES_GCM, 8). -define(CHACHA20_POLY1305, 9). +%% Following two are not defined in any RFC but we want to have the +%% same type of handling internaly, all of these "bulk_cipher_algorithm" +%% enums are only used internaly anyway. +-define(AES_CCM, 10). +-define(AES_CCM_8, 11). %% CipherType -define(STREAM, 0). diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index fde73cdef1..a05858221a 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -934,7 +934,7 @@ wait_sh(Type, Event, State) -> callback_mode() -> state_functions. -terminate({shutdown, sender_died, Reason}, _StateName, +terminate({shutdown, {sender_died, Reason}}, _StateName, #state{static_env = #static_env{socket = Socket, transport_cb = Transport}} = State) -> @@ -1119,7 +1119,7 @@ handle_info({CloseTag, Socket}, StateName, end; handle_info({'EXIT', Sender, Reason}, _, #state{protocol_specific = #{sender := Sender}} = State) -> - {stop, {shutdown, sender_died, Reason}, State}; + {stop, {shutdown, {sender_died, Reason}}, State}; handle_info(Msg, StateName, State) -> ssl_connection:StateName(info, Msg, State, ?MODULE). diff --git a/lib/ssl/src/tls_handshake_1_3.erl b/lib/ssl/src/tls_handshake_1_3.erl index 1e8b046c1e..0efedf3400 100644 --- a/lib/ssl/src/tls_handshake_1_3.erl +++ b/lib/ssl/src/tls_handshake_1_3.erl @@ -1007,7 +1007,8 @@ update_start_state(#state{connection_states = ConnectionStates0, session = Session#session{session_id = SessionId, ecc = Group, sign_alg = SelectedSignAlg, - dh_public_value = ClientPubKey}, + dh_public_value = ClientPubKey, + cipher_suite = Cipher}, connection_env = CEnv#connection_env{negotiated_version = {3,4}}}. diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl index 97331e1510..74321a1ae2 100644 --- a/lib/ssl/src/tls_record_1_3.erl +++ b/lib/ssl/src/tls_record_1_3.erl @@ -252,7 +252,7 @@ cipher_aead(Fragment, BulkCipherAlgo, Key, Seq, IV, TagLen) -> AAD = additional_data(erlang:iolist_size(Fragment) + TagLen), Nonce = nonce(Seq, IV), {Content, CipherTag} = - ssl_cipher:aead_encrypt(BulkCipherAlgo, Key, Nonce, Fragment, AAD), + ssl_cipher:aead_encrypt(BulkCipherAlgo, Key, Nonce, Fragment, AAD, TagLen), <<Content/binary, CipherTag/binary>>. encode_tls_cipher_text(#tls_cipher_text{opaque_type = Type, diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 57b74115ed..f7fae16088 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -43,6 +43,7 @@ MODULES = \ ssl_basic_SUITE \ ssl_bench_SUITE \ ssl_cipher_SUITE \ + ssl_cipher_suite_SUITE \ ssl_certificate_verify_SUITE\ ssl_crl_SUITE\ ssl_dist_SUITE \ @@ -63,8 +64,9 @@ MODULES = \ ssl_sni_SUITE \ ssl_eqc_SUITE \ ssl_rfc_5869_SUITE \ - make_certs\ - x509_test + make_certs \ + x509_test \ + inet_crypto_dist ERL_FILES = $(MODULES:%=%.erl) diff --git a/lib/ssl/test/inet_crypto_dist.erl b/lib/ssl/test/inet_crypto_dist.erl new file mode 100644 index 0000000000..5aafaac983 --- /dev/null +++ b/lib/ssl/test/inet_crypto_dist.erl @@ -0,0 +1,1323 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% ------------------------------------------------------------------------- +%% +%% Module for encrypted Erlang protocol - a minimal encrypted +%% distribution protocol based on only a shared secret +%% and the crypto application +%% +-module(inet_crypto_dist). +-define(DIST_NAME, inet_crypto). +-define(DIST_PROTO, crypto). +-define(DRIVER, inet_tcp). +-define(FAMILY, inet). + +-define(PROTOCOL, inet_crypto_dist_v1). +-define(DEFAULT_BLOCK_CRYPTO, aes_128_gcm). +-define(DEFAULT_HASH_ALGORITHM, sha256). +-define(DEFAULT_REKEY_INTERVAL, 32768). + +-export([listen/1, accept/1, accept_connection/5, + setup/5, close/1, select/1, is_node_name/1]). +-export([is_supported/0]). + +%% Generalized dist API, for sibling IPv6 module inet6_crypto_dist +-export([gen_listen/2, gen_accept/2, gen_accept_connection/6, + gen_setup/6, gen_close/2, gen_select/2]). + +-export([nodelay/0]). + +%% Debug +%%%-compile(export_all). +-export([dbg/0, test_server/0, test_client/1]). + +-include_lib("kernel/include/net_address.hrl"). +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). + +%% Test if crypto has got enough capabilities for this module to run +%% +is_supported() -> + try {crypto:cipher_info(?DEFAULT_BLOCK_CRYPTO), + crypto:hash_info(?DEFAULT_HASH_ALGORITHM)} + of + {#{block_size := _, iv_length := _, key_length := _}, + #{size := _}} -> + true + catch + error:undef -> + false + end. + +%% ------------------------------------------------------------------------- +%% Erlang distribution plugin structure explained to myself +%% ------- +%% These are the processes involved in the distribution: +%% * net_kernel +%% * The Acceptor +%% * The Controller | Handshaker | Ticker +%% * The DistCtrl process that may be split into: +%% + The Output controller +%% + The Input controller +%% For the regular inet_tcp_dist distribution module, DistCtrl +%% is not one or two processes, but one port - a gen_tcp socket +%% +%% When the VM is started with the argument "-proto_dist inet_crypto" +%% net_kernel registers the module inet_crypto_dist as distribution +%% module. net_kernel calls listen/1 to create a listen socket +%% and then accept/1 with the listen socket as argument to spawn +%% the Acceptor process, which is linked to net_kernel. Apparently +%% the listen socket is owned by net_kernel - I wonder if it could +%% be owned by the Acceptor process instead... +%% +%% The Acceptor process calls blocking accept on the listen socket +%% and when an incoming socket is returned it spawns the DistCtrl +%% process a linked to the Acceptor. The ownership of the accepted +%% socket is transferred to the DistCtrl process. +%% A message is sent to net_kernel to inform it that an incoming +%% connection has appeared and the Acceptor awaits a reply from net_kernel. +%% +%% net_kernel then calls accept_connection/5 to spawn the Controller | +%% Handshaker | Ticker process that is linked to net_kernel. +%% The Controller then awaits a message from the Acceptor process. +%% +%% When net_kernel has spawned the Controller it replies with a message +%% to the Acceptor that then calls DistCtrl to changes its links +%% so DistCtrl ends up linked to the Controller and not to the Acceptor. +%% The Acceptor then sends a message to the Controller. The Controller +%% then changes role into the Handshaker creates a #hs_data{} record +%% and calls dist_util:handshake_other_started/1. After this +%% the Acceptor goes back into a blocking accept on the listen socket. +%% +%% For the regular distribution inet_tcp_dist DistCtrl is a gen_tcp socket +%% and when it is a process it also acts as a socket. The #hs_data{} +%% record used by dist_util presents a set of funs that are used +%% by dist_util to perform the distribution handshake. These funs +%% make sure to transfer the handshake messages through the DistCtrl +%% "socket". +%% +%% When the handshake is finished a fun for this purpose in #hs_data{} +%% is called, which tells DistCtrl that it does not need to be prepared +%% for any more #hs_data{} handshake calls. The DistCtrl process in this +%% module then spawns the Input controller process that gets ownership +%% of the connection's gen_tcp socket and changes into {active, N} mode +%% so now it gets all incoming traffic and delivers that to the VM. +%% The original DistCtrl process changes role into the Output controller +%% process and starts asking the VM for outbound messages and transfers +%% them on the connection socket. +%% +%% The Handshaker now changes into the Ticker role, and uses only two +%% functions in the #hs_data{} record; one to get socket statistics +%% and one to send a tick. None of these may block for any reason +%% in particular not for a congested socket since that would destroy +%% connection supervision. +%% +%% +%% For an connection net_kernel calls setup/5 which spawns the +%% Controller process as linked to net_kernel. This Controller process +%% connects to the other node's listen socket and when that is succesful +%% spawns the DistCtrl process as linked to the controller and transfers +%% socket ownership to it. +%% +%% Then the Controller creates the #hs_data{} record and calls +%% dist_util:handshake_we_started/1 which changes the process role +%% into Handshaker. +%% +%% When the distribution handshake is finished the procedure is just +%% as for an incoming connection above. +%% +%% +%% To sum it up. +%% +%% There is an Acceptor process that is linked to net_kernel and +%% informs it when new connections arrive. +%% +%% net_kernel spawns Controllers for incoming and for outgoing connections. +%% these Controllers use the DistCtrl processes to do distribution +%% handshake and after that becomes Tickers that supervise the connection. +%% +%% The Controller | Handshaker | Ticker is linked to net_kernel, and to +%% DistCtrl, one or both. If any of these connection processes would die +%% all others should be killed by the links. Therefore none of them may +%% terminate with reason 'normal'. +%% ------------------------------------------------------------------------- + +%% ------------------------------------------------------------------------- +%% select/1 is called by net_kernel to ask if this distribution protocol +%% is willing to handle Node +%% + +select(Node) -> + gen_select(Node, ?DRIVER). + +gen_select(Node, Driver) -> + case dist_util:split_node(Node) of + {node, _, Host} -> + case Driver:getaddr(Host) of + {ok, _} -> true; + _ -> false + end; + _ -> + false + end. + +%% ------------------------------------------------------------------------- + +is_node_name(Node) -> + dist_util:is_node_name(Node). + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to create a listen socket for this +%% distribution protocol. This listen socket is used by +%% the Acceptor process. +%% + +listen(Name) -> + gen_listen(Name, ?DRIVER). + +gen_listen(Name, Driver) -> + case inet_tcp_dist:gen_listen(Driver, Name) of + {ok, {Socket, Address, Creation}} -> + inet:setopts(Socket, [binary, {nodelay, true}]), + {ok, + {Socket, Address#net_address{protocol = ?DIST_PROTO}, Creation}}; + Other -> + Other + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to spawn the Acceptor process that awaits +%% new connection in a blocking accept and informs net_kernel +%% when a new connection has appeared, and starts the DistCtrl +%% "socket" process for the connection. +%% + +accept(Listen) -> + gen_accept(Listen, ?DRIVER). + +gen_accept(Listen, Driver) -> + NetKernel = self(), + %% + %% Spawn Acceptor process + %% + Config = config(), + monitor_dist_proc( + spawn_opt( + fun () -> + accept_loop(Listen, Driver, NetKernel, Config) + end, + [link, {priority, max}])). + +accept_loop(Listen, Driver, NetKernel, Config) -> + case Driver:accept(Listen) of + {ok, Socket} -> + wait_for_code_server(), + Timeout = net_kernel:connecttime(), + DistCtrl = start_dist_ctrl(Socket, Config, Timeout), + %% DistCtrl is a "socket" + NetKernel ! + {accept, + self(), DistCtrl, Driver:family(), ?DIST_PROTO}, + receive + {NetKernel, controller, Controller} -> + call_dist_ctrl(DistCtrl, {controller, Controller, self()}), + Controller ! {self(), controller, Socket}; + {NetKernel, unsupported_protocol} -> + exit(unsupported_protocol) + end, + accept_loop(Listen, Driver, NetKernel, Config); + AcceptError -> + exit({accept, AcceptError}) + end. + +wait_for_code_server() -> + %% This is an ugly hack. Starting encryption on a connection + %% requires the crypto module to be loaded. Loading the crypto + %% module triggers its on_load function, which calls + %% code:priv_dir/1 to find the directory where its NIF library is. + %% However, distribution is started earlier than the code server, + %% so the code server is not necessarily started yet, and + %% code:priv_dir/1 might fail because of that, if we receive + %% an incoming connection on the distribution port early enough. + %% + %% If the on_load function of a module fails, the module is + %% unloaded, and the function call that triggered loading it fails + %% with 'undef', which is rather confusing. + %% + %% So let's avoid that by waiting for the code server to start. + %% + case whereis(code_server) of + undefined -> + timer:sleep(10), + wait_for_code_server(); + Pid when is_pid(Pid) -> + ok + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel when a new connection has appeared, to spawn +%% a Controller process that performs the handshake with the new node, +%% and then becomes the Ticker connection supervisor. +%% ------------------------------------------------------------------------- + +accept_connection(Acceptor, DistCtrl, MyNode, Allowed, SetupTime) -> + gen_accept_connection( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, ?DRIVER). + +gen_accept_connection( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, Driver) -> + NetKernel = self(), + %% + %% Spawn Controller/handshaker/ticker process + %% + monitor_dist_proc( + spawn_opt( + fun() -> + do_accept( + Acceptor, DistCtrl, + MyNode, Allowed, SetupTime, Driver, NetKernel) + end, + [link, {priority, max}])). + +do_accept( + Acceptor, DistCtrl, MyNode, Allowed, SetupTime, Driver, NetKernel) -> + receive + {Acceptor, controller, Socket} -> + Timer = dist_util:start_timer(SetupTime), + HSData = + hs_data_common( + NetKernel, MyNode, DistCtrl, Timer, + Socket, Driver:family()), + HSData_1 = + HSData#hs_data{ + this_node = MyNode, + this_flags = 0, + allowed = Allowed}, + dist_util:handshake_other_started(trace(HSData_1)) + end. + +%% ------------------------------------------------------------------------- +%% Called by net_kernel to spawn a Controller process that sets up +%% a new connection to another Erlang node, performs the handshake +%% with the other it, and then becomes the Ticker process +%% that supervises the connection. +%% ------------------------------------------------------------------------- + +setup(Node, Type, MyNode, LongOrShortNames, SetupTime) -> + gen_setup(Node, Type, MyNode, LongOrShortNames, SetupTime, ?DRIVER). + +gen_setup(Node, Type, MyNode, LongOrShortNames, SetupTime, Driver) -> + NetKernel = self(), + %% + %% Spawn Controller/handshaker/ticker process + %% + monitor_dist_proc( + spawn_opt( + setup_fun( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel), + [link, {priority, max}])). + +-spec setup_fun(_,_,_,_,_,_,_) -> fun(() -> no_return()). +setup_fun( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel) -> + fun() -> + do_setup( + Node, Type, MyNode, LongOrShortNames, SetupTime, + Driver, NetKernel) + end. + +-spec do_setup(_,_,_,_,_,_,_) -> no_return(). +do_setup( + Node, Type, MyNode, LongOrShortNames, SetupTime, Driver, NetKernel) -> + {Name, Address} = split_node(Driver, Node, LongOrShortNames), + ErlEpmd = net_kernel:epmd_module(), + {ARMod, ARFun} = get_address_resolver(ErlEpmd, Driver), + Timer = trace(dist_util:start_timer(SetupTime)), + case ARMod:ARFun(Name, Address, Driver:family()) of + {ok, Ip, TcpPort, Version} -> + do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version); + {ok, Ip} -> + case ErlEpmd:port_please(Name, Ip) of + {port, TcpPort, Version} -> + do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version); + Other -> + ?shutdown2( + Node, + trace( + {port_please_failed, ErlEpmd, Name, Ip, Other})) + end; + Other -> + ?shutdown2( + Node, + trace({getaddr_failed, Driver, Address, Other})) + end. + +-spec do_setup_connect(_,_,_,_,_,_,_,_,_) -> no_return(). + +do_setup_connect( + Node, Type, MyNode, Timer, Driver, NetKernel, + Ip, TcpPort, Version) -> + dist_util:reset_timer(Timer), + ConnectOpts = + trace( + connect_options( + [binary, {active, false}, {packet, 2}, {nodelay, true}])), + case Driver:connect(Ip, TcpPort, ConnectOpts) of + {ok, Socket} -> + Config = config(), + DistCtrl = + start_dist_ctrl(Socket, Config, net_kernel:connecttime()), + %% DistCtrl is a "socket" + HSData = + hs_data_common( + NetKernel, MyNode, DistCtrl, Timer, + Socket, Driver:family()), + HSData_1 = + HSData#hs_data{ + other_node = Node, + this_flags = 0, + other_version = Version, + request_type = Type}, + dist_util:handshake_we_started(trace(HSData_1)); + ConnectError -> + ?shutdown2(Node, + trace({connect_failed, Ip, TcpPort, ConnectError})) + end. + +%% ------------------------------------------------------------------------- +%% close/1 is only called by net_kernel on the socket returned by listen/1. + +close(Socket) -> + gen_close(Socket, ?DRIVER). + +gen_close(Socket, Driver) -> + trace(Driver:close(Socket)). + +%% ------------------------------------------------------------------------- + + +hs_data_common(NetKernel, MyNode, DistCtrl, Timer, Socket, Family) -> + %% Field 'socket' below is set to DistCtrl, which makes + %% the distribution handshake process (ticker) call + %% the funs below with DistCtrl as the S argument. + %% So, S =:= DistCtrl below... + #hs_data{ + kernel_pid = NetKernel, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + %% + f_send = + fun (S, Packet) when S =:= DistCtrl -> + call_dist_ctrl(S, {send, Packet}) + end, + f_recv = + fun (S, 0, infinity) when S =:= DistCtrl -> + case call_dist_ctrl(S, recv) of + {ok, Bin} when is_binary(Bin) -> + {ok, binary_to_list(Bin)}; + Error -> + Error + end + end, + f_setopts_pre_nodeup = + fun (S) when S =:= DistCtrl -> + ok + end, + f_setopts_post_nodeup = + fun (S) when S =:= DistCtrl -> + ok + end, + f_getll = + fun (S) when S =:= DistCtrl -> + {ok, S} %% DistCtrl is the distribution port + end, + f_address = + fun (S, Node) when S =:= DistCtrl -> + case call_dist_ctrl(S, peername) of + {ok, Address} -> + case dist_util:split_node(Node) of + {node, _, Host} -> + #net_address{ + address = Address, + host = Host, + protocol = ?DIST_PROTO, + family = Family}; + _ -> + {error, no_node} + end + end + end, + f_handshake_complete = + fun (S, _Node, DistHandle) when S =:= DistCtrl -> + call_dist_ctrl(S, {handshake_complete, DistHandle}) + end, + %% + %% mf_tick/1, mf_getstat/1, mf_setopts/2 and mf_getopts/2 + %% are called by the ticker any time after f_handshake_complete/3 + %% so they may not block the caller even for congested socket + mf_tick = + fun (S) when S =:= DistCtrl -> + S ! dist_tick + end, + mf_getstat = + fun (S) when S =:= DistCtrl -> + case + inet:getstat(Socket, [recv_cnt, send_cnt, send_pend]) + of + {ok, Stat} -> + split_stat(Stat, 0, 0, 0); + Error -> + Error + end + end, + mf_setopts = + fun (S, Opts) when S =:= DistCtrl -> + inet:setopts(Socket, setopts_filter(Opts)) + end, + mf_getopts = + fun (S, Opts) when S =:= DistCtrl -> + inet:getopts(Socket, Opts) + end}. + +setopts_filter(Opts) -> + [Opt || + Opt <- Opts, + case Opt of + {K, _} when K =:= active; K =:= deliver; K =:= packet -> false; + K when K =:= list; K =:= binary -> false; + K when K =:= inet; K =:= inet6 -> false; + _ -> true + end]. + +split_stat([{recv_cnt, R}|Stat], _, W, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_cnt, W}|Stat], R, _, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_pend, P}|Stat], R, W, _) -> + split_stat(Stat, R, W, P); +split_stat([], R, W, P) -> + {ok, R, W, P}. + +%% ------------------------------------------------------------ +%% Determine if EPMD module supports address resolving. Default +%% is to use inet_tcp:getaddr/2. +%% ------------------------------------------------------------ +get_address_resolver(EpmdModule, _Driver) -> + case erlang:function_exported(EpmdModule, address_please, 3) of + true -> {EpmdModule, address_please}; + _ -> {erl_epmd, address_please} + end. + + +%% If Node is illegal terminate the connection setup!! +split_node(Driver, Node, LongOrShortNames) -> + case dist_util:split_node(Node) of + {node, Name, Host} -> + check_node(Driver, Node, Name, Host, LongOrShortNames); + {host, _} -> + error_logger:error_msg( + "** Nodename ~p illegal, no '@' character **~n", + [Node]), + ?shutdown2(Node, trace({illegal_node_n@me, Node})); + _ -> + error_logger:error_msg( + "** Nodename ~p illegal **~n", [Node]), + ?shutdown2(Node, trace({illegal_node_name, Node})) + end. + +check_node(Driver, Node, Name, Host, LongOrShortNames) -> + case string:split(Host, ".", all) of + [_] when LongOrShortNames =:= longnames -> + case Driver:parse_address(Host) of + {ok, _} -> + {Name, Host}; + _ -> + error_logger:error_msg( + "** System running to use " + "fully qualified hostnames **~n" + "** Hostname ~s is illegal **~n", + [Host]), + ?shutdown2(Node, trace({not_longnames, Host})) + end; + [_, _|_] when LongOrShortNames =:= shortnames -> + error_logger:error_msg( + "** System NOT running to use " + "fully qualified hostnames **~n" + "** Hostname ~s is illegal **~n", + [Host]), + ?shutdown2(Node, trace({not_shortnames, Host})); + _ -> + {Name, Host} + end. + +%% ------------------------------------------------------------------------- + +connect_options(Opts) -> + case application:get_env(kernel, inet_dist_connect_options) of + {ok, ConnectOpts} -> + Opts ++ setopts_filter(ConnectOpts); + _ -> + Opts + end. + +%% we may not always want the nodelay behaviour +%% for performance reasons +nodelay() -> + case application:get_env(kernel, dist_nodelay) of + undefined -> + {nodelay, true}; + {ok, true} -> + {nodelay, true}; + {ok, false} -> + {nodelay, false}; + _ -> + {nodelay, true} + end. + +config() -> + case init:get_argument(?DIST_NAME) of + error -> + error({missing_argument, ?DIST_NAME}); + {ok, [[String]]} -> + {ok, Tokens, _} = erl_scan:string(String ++ "."), + case erl_parse:parse_term(Tokens) of + {ok, #{secret := Secret} = Config} + when is_binary(Secret); is_list(Secret) -> + Config; + {ok, #{} = Config} -> + error({missing_secret, [{?DIST_NAME,Config}]}); + _ -> + error({bad_argument_value, [{?DIST_NAME,String}]}) + end; + {ok, Value} -> + error({malformed_argument, [{?DIST_NAME,Value}]}) + end. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% +%% The DistCtrl process(es). +%% +%% At net_kernel handshake_complete spawns off the input controller that +%% takes over the socket ownership, and itself becomes the output controller +%% +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%% XXX Missing to "productified": +%%% * Cryptoanalysis by experts +%%% * Proof of usefulness +%%% * Unifying exit reasons using a death_row() function +%%% * Verification (and rejection) of other end's crypto parameters +%%% * OTP:ification (proc_lib?) +%%% * An application to belong to (crypto|kernel?) +%%% * Secret on file (cookie as default?), parameter handling +%%% * Restart and/or code reload policy + +%% Debug client and server + +test_config() -> + #{secret => <<"Secret Cluster Password 123456">>}. + +test_server() -> + {ok, Listen} = gen_tcp:listen(0, [{packet, 2}, {active, false}, binary]), + {ok, Port} = inet:port(Listen), + io:format(?MODULE_STRING":test_client(~w).~n", [Port]), + {ok, Socket} = gen_tcp:accept(Listen), + test(Socket). + +test_client(Port) -> + {ok, Socket} = + gen_tcp:connect( + localhost, Port, [{packet, 2}, {active, false}, binary]), + test(Socket). + +test(Socket) -> + start_dist_ctrl(Socket, test_config(), 10000). + +%% ------------------------------------------------------------------------- + +start_dist_ctrl(Socket, Config, Timeout) -> + Protocol = ?PROTOCOL, + Controller = self(), + Server = + monitor_dist_proc( + spawn_opt( + fun () -> + receive + {?MODULE, From, start} -> + {SendParams, RecvParams} = + init(Socket, Config, Protocol), + reply(From, self()), + handshake(SendParams, 0, RecvParams, 0, Controller) + end + end, + [link, + {priority, max}, + {message_queue_data, off_heap}, + {fullsweep_after, 0}])), + ok = gen_tcp:controlling_process(Socket, Server), + call_dist_ctrl(Server, start, Timeout). + + +call_dist_ctrl(Server, Msg) -> + call_dist_ctrl(Server, Msg, infinity). +%% +call_dist_ctrl(Server, Msg, Timeout) -> + Ref = erlang:monitor(process, Server), + Server ! {?MODULE, {Ref, self()}, Msg}, + receive + {Ref, Res} -> + erlang:demonitor(Ref, [flush]), + Res; + {'DOWN', Ref, process, Server, Reason} -> + exit({?PROTOCOL, Reason}) + after Timeout -> + exit(Server, timeout), + receive + {'DOWN', Ref, process, Server, _} -> + exit({?PROTOCOL, timeout}) + end + end. + +reply({Ref, Pid}, Msg) -> + Pid ! {Ref, Msg}, + ok. + +%% ------------------------------------------------------------------------- + +-record(params, + {protocol, % Encryption protocol tag + socket, + dist_handle, + hash_algorithm, + block_crypto, + rekey_interval, + iv, + key, + tag_len}). + +-define(TCP_ACTIVE, 64). +-define(CHUNK_SIZE, (65536 - 512)). +%% The start chunk starts with zeros, so it seems logical to not have +%% a chunk type with value 0 +-define(HANDSHAKE_CHUNK, 1). +-define(DATA_CHUNK, 2). +-define(TICK_CHUNK, 3). +-define(REKEY_CHUNK, 4). + +%% ------------------------------------------------------------------------- +%% Crypto strategy +%% ------- +%% The crypto strategy is as simple as possible to get an encrypted +%% connection as benchmark reference. It is geared around AEAD +%% ciphers in particular AES-GCM. +%% +%% The init message and the start message must fit in the TCP buffers +%% since both sides start with sending the init message, waits +%% for the other end's init message, sends the start message +%% and waits for the other end's start message. So if the send +%% blocks we have a deadlock. +%% +%% The init message is unencrypted and contains the block cipher and hash +%% algorithms the sender will use, the IV and a key salt. Both sides' +%% key salt is used with the mutual secret as input to the hash algorithm +%% to create different encryption/decryption keys for both directions. +%% +%% The start message is the first encrypted message and contains just +%% encrypted zeros the width of the key, with the header of the init +%% message as AAD data. Successfully decrypting this message +%% verifies that we have an encrypted channel. +%% +%% Subsequent encrypted messages has the sequence number and the length +%% of the message as AAD data. These messages has got a message type +%% differentiating data from ticks. Ticks have a random size in an +%% attempt to make them less obvious to spot. +%% +%% The only reaction to errors is to crash noisily wich will bring +%% down the connection and hopefully produce something useful +%% in the local log, but all the other end sees is a closed connection. +%% ------------------------------------------------------------------------- + +init(Socket, Config, Protocol) -> + Secret = maps:get(secret, Config), + HashAlgorithm = + maps:get(hash_algorithm, Config, ?DEFAULT_HASH_ALGORITHM), + BlockCrypto = + maps:get(block_crypto, Config, ?DEFAULT_BLOCK_CRYPTO), + RekeyInterval = + maps:get(rekey_interval, Config, ?DEFAULT_REKEY_INTERVAL), + %% + SendParams = + init_params( + Socket, Protocol, HashAlgorithm, BlockCrypto, RekeyInterval), + send_init(SendParams, Secret). + +send_init( + #params{ + protocol = Protocol, + socket = Socket, + block_crypto = BlockCrypto, + iv = IVLen, + key = KeyLen, + hash_algorithm = HashAlgorithm} = SendParams, + Secret) -> + %% + ProtocolString = atom_to_binary(Protocol, utf8), + BlockCryptoString = atom_to_binary(BlockCrypto, utf8), + HashAlgorithmString = atom_to_binary(HashAlgorithm, utf8), + SendHeader = + <<ProtocolString/binary, 0, + HashAlgorithmString/binary, 0, + BlockCryptoString/binary, 0>>, + <<IV:IVLen/binary, KeySalt:KeyLen/binary>> = IV_KeySalt = + crypto:strong_rand_bytes(IVLen + KeyLen), + InitPacket = [SendHeader, IV_KeySalt], + ok = gen_tcp:send(Socket, InitPacket), + recv_init(SendParams#params{iv = IV, key = KeySalt}, Secret, SendHeader). + +recv_init( + #params{ + socket = Socket, + hash_algorithm = SendHashAlgorithm, + key = SendKeySalt} = SendParams, Secret, SendHeader) -> + %% + {ok, InitPacket} = gen_tcp:recv(Socket, 0), + [ProtocolString, Rest_1] = binary:split(InitPacket, <<0>>), + Protocol = binary_to_existing_atom(ProtocolString, utf8), + case Protocol of + ?PROTOCOL -> + [HashAlgorithmString, Rest_2] = binary:split(Rest_1, <<0>>), + HashAlgorithm = binary_to_existing_atom(HashAlgorithmString, utf8), + [BlockCryptoString, Rest_3] = binary:split(Rest_2, <<0>>), + BlockCrypto = binary_to_existing_atom(BlockCryptoString, utf8), + #params{ + hash_algorithm = RecvHashAlgorithm, + iv = RecvIVLen, + key = RecvKeyLen} = RecvParams = + init_params( + Socket, Protocol, HashAlgorithm, BlockCrypto, undefined), + <<RecvIV:RecvIVLen/binary, + RecvKeySalt:RecvKeyLen/binary>> = Rest_3, + SendKey = + hash_key(SendHashAlgorithm, SendKeySalt, [RecvKeySalt, Secret]), + RecvKey = + hash_key(RecvHashAlgorithm, RecvKeySalt, [SendKeySalt, Secret]), + SendParams_1 = SendParams#params{key = SendKey}, + RecvParams_1 = RecvParams#params{iv = RecvIV, key = RecvKey}, + RecvHeaderLen = byte_size(InitPacket) - RecvIVLen - RecvKeyLen, + <<RecvHeader:RecvHeaderLen/binary, _/binary>> = InitPacket, + send_start(SendParams_1, SendHeader), + RecvRekeyInterval = recv_start(RecvParams_1, RecvHeader), + {SendParams_1, + RecvParams_1#params{rekey_interval = RecvRekeyInterval}} + end. + +send_start( + #params{ + socket = Socket, + block_crypto = BlockCrypto, + rekey_interval= RekeyInterval, + iv = IV, + key = Key, + tag_len = TagLen}, AAD) -> + %% + KeyLen = byte_size(Key), + Zeros = binary:copy(<<0>>, KeyLen), + {Ciphertext, CipherTag} = + crypto:block_encrypt( + crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, [Zeros, <<RekeyInterval:32>>], TagLen}), + ok = gen_tcp:send(Socket, [Ciphertext, CipherTag]). + +recv_start( + #params{ + socket = Socket, + block_crypto = BlockCrypto, + iv = IV, + key = Key, + tag_len = TagLen}, AAD) -> + {ok, Packet} = gen_tcp:recv(Socket, 0), + KeyLen = byte_size(Key), + PacketLen = KeyLen + 4, + <<Ciphertext:PacketLen/binary, CipherTag:TagLen/binary>> = Packet, + Zeros = binary:copy(<<0>>, KeyLen), + case + crypto:block_decrypt( + crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, Ciphertext, CipherTag}) + of + <<Zeros:KeyLen/binary, RekeyInterval:32>> + when 1 =< RekeyInterval -> + RekeyInterval; + _ -> + error(decrypt_error) + end. + +init_params(Socket, Protocol, HashAlgorithm, BlockCrypto, RekeyInterval) -> + #{block_size := 1, + iv_length := IVLen, + key_length := KeyLen} = crypto:cipher_info(BlockCrypto), + case crypto:hash_info(HashAlgorithm) of + #{size := HashSize} when HashSize >= KeyLen -> + #params{ + socket = Socket, + protocol = Protocol, + hash_algorithm = HashAlgorithm, + block_crypto = BlockCrypto, + rekey_interval = RekeyInterval, + iv = IVLen, + key = KeyLen, + tag_len = 16} + end. + +crypto_cipher_name(BlockCrypto) -> + case BlockCrypto of + aes_128_gcm -> aes_gcm; + aes_192_gcm -> aes_gcm; + aes_256_gcm -> aes_gcm + end. + +hash_key(HashAlgorithm, KeySalt, OtherSalt) -> + KeyLen = byte_size(KeySalt), + <<Key:KeyLen/binary, _/binary>> = + crypto:hash(HashAlgorithm, [KeySalt, OtherSalt]), + Key. + +%% ------------------------------------------------------------------------- +%% net_kernel distribution handshake in progress +%% + +handshake( + SendParams, SendSeq, + #params{socket = Socket} = RecvParams, RecvSeq, Controller) -> + receive + {?MODULE, From, {controller, Controller_1, Parent}} -> + Result = link(Controller_1), + true = unlink(Parent), + reply(From, Result), + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller_1); + {?MODULE, From, {handshake_complete, DistHandle}} -> + reply(From, ok), + InputHandler = + monitor_dist_proc( + spawn_opt( + fun () -> + link(Controller), + receive + DistHandle -> + ok = + inet:setopts( + Socket, + [{active, ?TCP_ACTIVE}, + nodelay()]), + input_handler( + RecvParams#params{ + dist_handle = DistHandle}, + RecvSeq, empty_q(), infinity) + end + end, + [link, + {priority, normal}, + {message_queue_data, off_heap}, + {fullsweep_after, 0}])), + _ = monitor(process, InputHandler), % For the benchmark test + ok = gen_tcp:controlling_process(Socket, InputHandler), + ok = erlang:dist_ctrl_input_handler(DistHandle, InputHandler), + InputHandler ! DistHandle, + process_flag(priority, normal), + erlang:dist_ctrl_get_data_notification(DistHandle), + crypto:rand_seed_alg(crypto_cache), + output_handler( + SendParams#params{dist_handle = DistHandle}, SendSeq); + %% + {?MODULE, From, {send, Data}} -> + {SendParams_1, SendSeq_1} = + encrypt_and_send_chunk( + SendParams, SendSeq, [?HANDSHAKE_CHUNK, Data]), + reply(From, ok), + handshake( + SendParams_1, SendSeq_1, RecvParams, RecvSeq, Controller); + {?MODULE, From, recv} -> + {RecvParams_1, RecvSeq_1, Reply} = + recv_and_decrypt_chunk(RecvParams, RecvSeq), + reply(From, Reply), + handshake( + SendParams, SendSeq, RecvParams_1, RecvSeq_1, Controller); + {?MODULE, From, peername} -> + reply(From, inet:peername(Socket)), + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller); + %% + _Alien -> + handshake(SendParams, SendSeq, RecvParams, RecvSeq, Controller) + end. + +recv_and_decrypt_chunk(#params{socket = Socket} = RecvParams, RecvSeq) -> + case gen_tcp:recv(Socket, 0) of + {ok, Chunk} -> + case decrypt_chunk(RecvParams, RecvSeq, Chunk) of + <<?HANDSHAKE_CHUNK, Cleartext/binary>> -> + {RecvParams, RecvSeq + 1, {ok, Cleartext}}; + #params{} = RecvParams_1 -> + recv_and_decrypt_chunk(RecvParams_1, 0); + _ -> + error(decrypt_error) + end; + Error -> + {RecvParams, RecvSeq, Error} + end. + +%% ------------------------------------------------------------------------- +%% Output handler process +%% +%% The game here is to flush all dist_data and dist_tick messages, +%% prioritize dist_data over dist_tick, and to not use selective receive + +output_handler(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_tick(Params, Seq); + _Other -> + %% Ignore + output_handler(Params, Seq) + end + end. + +output_handler_data(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_data(Params, Seq); + _Other -> + %% Ignore + output_handler_data(Params, Seq) + end + after 0 -> + DistHandle = Params#params.dist_handle, + Q = get_data(DistHandle, empty_q()), + {Params_1, Seq_1} = output_handler_send(Params, Seq, Q, true), + erlang:dist_ctrl_get_data_notification(DistHandle), + output_handler(Params_1, Seq_1) + end. + +output_handler_tick(Params, Seq) -> + receive + Msg -> + case Msg of + dist_data -> + output_handler_data(Params, Seq); + dist_tick -> + output_handler_tick(Params, Seq); + _Other -> + %% Ignore + output_handler_tick(Params, Seq) + end + after 0 -> + TickSize = 8 + rand:uniform(56), + TickData = binary:copy(<<0>>, TickSize), + {Params_1, Seq_1} = + encrypt_and_send_chunk(Params, Seq, [?TICK_CHUNK, TickData]), + output_handler(Params_1, Seq_1) + end. + +output_handler_send( + #params{dist_handle = DistHandle} = Params, Seq, {_, Size, _} = Q, Retry) -> + %% + if + ?CHUNK_SIZE < Size -> + {Cleartext, Q_1} = deq_iovec(?CHUNK_SIZE, Q), + {Params_1, Seq_1} = + encrypt_and_send_chunk(Params, Seq, [?DATA_CHUNK, Cleartext]), + output_handler_send(Params_1, Seq_1, Q_1, Retry); + Retry -> + Q_1 = get_data(DistHandle, Q), + output_handler_send(Params, Seq, Q_1, false); + true -> + {Cleartext, _} = deq_iovec(Size, Q), + encrypt_and_send_chunk(Params, Seq, [?DATA_CHUNK, Cleartext]) + end. + +%% ------------------------------------------------------------------------- +%% Input handler process +%% +%% Here is T 0 or infinity to steer if we should try to receive +%% more data or not; start with infinity, and when we get some +%% data try with 0 to see if more is waiting + +input_handler(#params{socket = Socket} = Params, Seq, Q, T) -> + receive + Msg -> + case Msg of + {tcp_passive, Socket} -> + ok = inet:setopts(Socket, [{active, ?TCP_ACTIVE}]), + Q_1 = + case T of + 0 -> + deliver_data(Params#params.dist_handle, Q); + infinity -> + Q + end, + input_handler(Params, Seq, Q_1, infinity); + {tcp, Socket, Chunk} -> + input_chunk(Params, Seq, Q, Chunk); + {tcp_closed, Socket} -> + error(connection_closed); + _Other -> + %% Ignore... + input_handler(Params, Seq, Q, T) + end + after T -> + Q_1 = deliver_data(Params#params.dist_handle, Q), + input_handler(Params, Seq, Q_1, infinity) + end. + +input_chunk(Params, Seq, Q, Chunk) -> + case decrypt_chunk(Params, Seq, Chunk) of + <<?DATA_CHUNK, Cleartext/binary>> -> + input_handler(Params, Seq + 1, enq_binary(Cleartext, Q), 0); + <<?TICK_CHUNK, _/binary>> -> + input_handler(Params, Seq + 1, Q, 0); + #params{} = Params_1 -> + input_handler(Params_1, 0, Q, 0); + _ -> + error(decrypt_error) + end. + +%% ------------------------------------------------------------------------- +%% erlang:dist_ctrl_* helpers + +%% Get data for sending from the VM and place it in a queue +%% +get_data(DistHandle, {Front, Size, Rear}) -> + get_data(DistHandle, Front, Size, Rear). +%% +get_data(DistHandle, Front, Size, Rear) -> + case erlang:dist_ctrl_get_data(DistHandle) of + none -> + {Front, Size, Rear}; + Bin when is_binary(Bin) -> + Len = byte_size(Bin), + get_data( + DistHandle, Front, Size + 4 + Len, + [Bin, <<Len:32>>|Rear]); + [Bin1, Bin2] -> + Len = byte_size(Bin1) + byte_size(Bin2), + get_data( + DistHandle, Front, Size + 4 + Len, + [Bin2, Bin1, <<Len:32>>|Rear]); + Iovec -> + Len = iolist_size(Iovec), + get_data( + DistHandle, Front, Size + 4 + Len, + lists:reverse(Iovec, [<<Len:32>>|Rear])) + end. + +%% De-packet and deliver received data to the VM from a queue +%% +deliver_data(DistHandle, Q) -> + case Q of + {[], Size, []} -> + Size = 0, % Assert + Q; + {[], Size, Rear} -> + [Bin|Front] = lists:reverse(Rear), + deliver_data(DistHandle, Front, Size, [], Bin); + {[Bin|Front], Size, Rear} -> + deliver_data(DistHandle, Front, Size, Rear, Bin) + end. +%% +deliver_data(DistHandle, Front, Size, Rear, Bin) -> + case Bin of + <<DataSizeA:32, DataA:DataSizeA/binary, + DataSizeB:32, DataB:DataSizeB/binary, Rest/binary>> -> + erlang:dist_ctrl_put_data(DistHandle, DataA), + erlang:dist_ctrl_put_data(DistHandle, DataB), + deliver_data( + DistHandle, + Front, Size - (4 + DataSizeA + 4 + DataSizeB), Rear, + Rest); + <<DataSize:32, Data:DataSize/binary, Rest/binary>> -> + erlang:dist_ctrl_put_data(DistHandle, Data), + deliver_data(DistHandle, Front, Size - (4 + DataSize), Rear, Rest); + <<DataSize:32, FirstData/binary>> -> + TotalSize = 4 + DataSize, + if + TotalSize =< Size -> + BinSize = byte_size(Bin), + {MoreData, Q} = + deq_iovec( + TotalSize - BinSize, + Front, Size - BinSize, Rear), + erlang:dist_ctrl_put_data(DistHandle, [FirstData|MoreData]), + deliver_data(DistHandle, Q); + true -> % Incomplete data + {[Bin|Front], Size, Rear} + end; + <<_/binary>> -> + BinSize = byte_size(Bin), + if + 4 =< Size -> % Fragmented header - extract a header bin + {RestHeader, {Front_1, _Size_1, Rear_1}} = + deq_iovec(4 - BinSize, Front, Size - BinSize, Rear), + Header = iolist_to_binary([Bin|RestHeader]), + deliver_data(DistHandle, Front_1, Size, Rear_1, Header); + true -> % Incomplete header + {[Bin|Front], Size, Rear} + end + end. + +%% ------------------------------------------------------------------------- +%% Encryption and decryption helpers + +encrypt_and_send_chunk( + #params{ + socket = Socket, rekey_interval = Seq, + key = Key, iv = IV, hash_algorithm = HashAlgorithm} = Params, + Seq, Cleartext) -> + %% + KeyLen = byte_size(Key), + IVLen = byte_size(IV), + Chunk = <<IV_1:IVLen/binary, KeySalt:KeyLen/binary>> = + crypto:strong_rand_bytes(IVLen + KeyLen), + ok = gen_tcp:send(Socket, encrypt_chunk(Params, Seq, [?REKEY_CHUNK, Chunk])), + Key_1 = hash_key(HashAlgorithm, Key, KeySalt), + Params_1 = Params#params{key = Key_1, iv = IV_1}, + ok = gen_tcp:send(Socket, encrypt_chunk(Params_1, 0, Cleartext)), + {Params_1, 1}; +encrypt_and_send_chunk(#params{socket = Socket} = Params, Seq, Cleartext) -> + ok = gen_tcp:send(Socket, encrypt_chunk(Params, Seq, Cleartext)), + {Params, Seq + 1}. + +encrypt_chunk( + #params{ + block_crypto = BlockCrypto, + iv = IV, key = Key, tag_len = TagLen}, Seq, Cleartext) -> + %% + ChunkLen = iolist_size(Cleartext) + TagLen, + AAD = <<Seq:32, ChunkLen:32>>, + {Ciphertext, CipherTag} = + crypto:block_encrypt( + crypto_cipher_name(BlockCrypto), Key, IV, {AAD, Cleartext, TagLen}), + Chunk = [Ciphertext,CipherTag], + Chunk. + +decrypt_chunk( + #params{ + block_crypto = BlockCrypto, + iv = IV, key = Key, tag_len = TagLen} = Params, Seq, Chunk) -> + %% + ChunkLen = byte_size(Chunk), + true = TagLen =< ChunkLen, % Assert + AAD = <<Seq:32, ChunkLen:32>>, + CiphertextLen = ChunkLen - TagLen, + <<Ciphertext:CiphertextLen/binary, CipherTag:TagLen/binary>> = Chunk, + block_decrypt( + Params, Seq, crypto_cipher_name(BlockCrypto), + Key, IV, {AAD, Ciphertext, CipherTag}). + +block_decrypt( + #params{rekey_interval = Seq} = Params, Seq, CipherName, Key, IV, Data) -> + %% + KeyLen = byte_size(Key), + IVLen = byte_size(IV), + case crypto:block_decrypt(CipherName, Key, IV, Data) of + <<?REKEY_CHUNK, IV_1:IVLen/binary, KeySalt:KeyLen/binary>> -> + Key_1 = hash_key(Params#params.hash_algorithm, Key, KeySalt), + Params#params{iv = IV_1, key = Key_1}; + _ -> + error(decrypt_error) + end; +block_decrypt(_Params, _Seq, CipherName, Key, IV, Data) -> + crypto:block_decrypt(CipherName, Key, IV, Data). + +%% ------------------------------------------------------------------------- +%% Queue of binaries i.e an iovec queue + +empty_q() -> + {[], 0, []}. + +enq_binary(Bin, {Front, Size, Rear}) -> + {Front, Size + byte_size(Bin), [Bin|Rear]}. + +deq_iovec(GetSize, {Front, Size, Rear}) when GetSize =< Size -> + deq_iovec(GetSize, Front, Size, Rear, []). +%% +deq_iovec(GetSize, Front, Size, Rear) -> + deq_iovec(GetSize, Front, Size, Rear, []). +%% +deq_iovec(GetSize, [], Size, Rear, Acc) -> + deq_iovec(GetSize, lists:reverse(Rear), Size, [], Acc); +deq_iovec(GetSize, [Bin|Front], Size, Rear, Acc) -> + BinSize = byte_size(Bin), + if + BinSize < GetSize -> + deq_iovec( + GetSize - BinSize, Front, Size - BinSize, Rear, [Bin|Acc]); + GetSize < BinSize -> + {Bin1,Bin2} = erlang:split_binary(Bin, GetSize), + {lists:reverse(Acc, [Bin1]), {[Bin2|Front], Size - GetSize, Rear}}; + true -> + {lists:reverse(Acc, [Bin]), {Front, Size - BinSize, Rear}} + end. + +%% ------------------------------------------------------------------------- + +%% Trace point +trace(Term) -> Term. + +%% Keep an eye on this Pid (debug) +monitor_dist_proc(Pid) -> +%%% spawn( +%%% fun () -> +%%% MRef = erlang:monitor(process, Pid), +%%% receive +%%% {'DOWN', MRef, _, _, normal} -> +%%% error_logger:error_report( +%%% [dist_proc_died, +%%% {reason, normal}, +%%% {pid, Pid}]); +%%% {'DOWN', MRef, _, _, Reason} -> +%%% error_logger:info_report( +%%% [dist_proc_died, +%%% {reason, Reason}, +%%% {pid, Pid}]) +%%% end +%%% end), + Pid. + +dbg() -> + dbg:stop(), + dbg:tracer(), + dbg:p(all, c), + dbg:tpl(?MODULE, cx), + dbg:tpl(erlang, dist_ctrl_get_data_notification, cx), + dbg:tpl(erlang, dist_ctrl_get_data, cx), + dbg:tpl(erlang, dist_ctrl_put_data, cx), + ok. diff --git a/lib/ssl/test/ssl.spec b/lib/ssl/test/ssl.spec index 24272327c3..15587abecd 100644 --- a/lib/ssl/test/ssl.spec +++ b/lib/ssl/test/ssl.spec @@ -6,5 +6,7 @@ {skip_groups,dir,ssl_bench_SUITE,payload,"Benchmarks run separately"}. {skip_groups,dir,ssl_bench_SUITE,pem_cache,"Benchmarks run separately"}. {skip_groups,dir,ssl_dist_bench_SUITE,setup,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,roundtrip,"Benchmarks run separately"}. {skip_groups,dir,ssl_dist_bench_SUITE,throughput,"Benchmarks run separately"}. +{skip_groups,dir,ssl_dist_bench_SUITE,sched_utilization,"Benchmarks run separately"}. diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index ff5638ff34..03ee97de5d 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -76,11 +76,9 @@ groups() -> {'sslv3', [], all_versions_groups() ++ tls_versions_groups() ++ rizzo_tests() ++ [tls_ciphersuite_vs_version]}, {api,[], api_tests()}, {api_tls,[], api_tests_tls()}, - {tls_ciphers,[], tls_cipher_tests()}, {session, [], session_tests()}, {renegotiate, [], renegotiate_tests()}, {ciphers, [], cipher_tests()}, - {ciphers_ec, [], cipher_tests_ec()}, {error_handling_tests, [], error_handling_tests()}, {error_handling_tests_tls, [], error_handling_tests_tls()} ]. @@ -88,14 +86,12 @@ groups() -> tls_versions_groups ()-> [ {group, api_tls}, - {group, tls_ciphers}, {group, error_handling_tests_tls}]. all_versions_groups ()-> [{group, api}, {group, renegotiate}, {group, ciphers}, - {group, ciphers_ec}, {group, error_handling_tests}]. @@ -211,38 +207,11 @@ renegotiate_tests() -> renegotiate_dos_mitigate_passive, renegotiate_dos_mitigate_absolute]. -tls_cipher_tests() -> - [rc4_rsa_cipher_suites, - rc4_ecdh_rsa_cipher_suites, - rc4_ecdsa_cipher_suites]. - cipher_tests() -> [old_cipher_suites, - cipher_suites_mix, - ciphers_rsa_signed_certs, - ciphers_rsa_signed_certs_openssl_names, - ciphers_dsa_signed_certs, - ciphers_dsa_signed_certs_openssl_names, - chacha_rsa_cipher_suites, - chacha_ecdsa_cipher_suites, - anonymous_cipher_suites, - psk_cipher_suites, - psk_with_hint_cipher_suites, - psk_anon_cipher_suites, - psk_anon_with_hint_cipher_suites, - srp_cipher_suites, - srp_anon_cipher_suites, - srp_dsa_cipher_suites, - des_rsa_cipher_suites, - des_ecdh_rsa_cipher_suites, + cipher_suites_mix, default_reject_anonymous]. -cipher_tests_ec() -> - [ciphers_ecdsa_signed_certs, - ciphers_ecdsa_signed_certs_openssl_names, - ciphers_ecdh_rsa_signed_certs, - ciphers_ecdh_rsa_signed_certs_openssl_names]. - error_handling_tests()-> [close_transport_accept, recv_active, @@ -289,7 +258,8 @@ tls13_test_group() -> tls13_hrr_client_auth_empty_cert_ssl_server_openssl_client, tls13_hrr_client_auth_ssl_server_openssl_client, tls13_unsupported_sign_algo_client_auth_ssl_server_openssl_client, - tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client]. + tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client, + tls13_connection_information]. %%-------------------------------------------------------------------- init_per_suite(Config0) -> @@ -409,26 +379,7 @@ init_per_testcase(TestCase, Config) when TestCase == client_renegotiate; ct:timetrap({seconds, ?SEC_RENEGOTIATION_TIMEOUT + 5}), Config; -init_per_testcase(TestCase, Config) when TestCase == psk_cipher_suites; - TestCase == psk_with_hint_cipher_suites; - TestCase == ciphers_rsa_signed_certs; - TestCase == ciphers_rsa_signed_certs_openssl_names; - TestCase == ciphers_ecdh_rsa_signed_certs_openssl_names; - TestCase == ciphers_ecdh_rsa_signed_certs; - TestCase == ciphers_dsa_signed_certs; - TestCase == ciphers_dsa_signed_certs_openssl_names; - TestCase == anonymous_cipher_suites; - TestCase == ciphers_ecdsa_signed_certs; - TestCase == ciphers_ecdsa_signed_certs_openssl_names; - TestCase == anonymous_cipher_suites; - TestCase == psk_anon_cipher_suites; - TestCase == psk_anon_with_hint_cipher_suites; - TestCase == srp_cipher_suites; - TestCase == srp_anon_cipher_suites; - TestCase == srp_dsa_cipher_suites; - TestCase == des_rsa_cipher_suites; - TestCase == des_ecdh_rsa_cipher_suites; - TestCase == versions_option; +init_per_testcase(TestCase, Config) when TestCase == versions_option; TestCase == tls_tcp_connect_big -> ssl_test_lib:ct_log_supported_protocol_versions(Config), ct:timetrap({seconds, 60}), @@ -2708,144 +2659,6 @@ tls_shutdown_error(Config) when is_list(Config) -> ok = ssl:close(Listen), {error, closed} = ssl:shutdown(Listen, read_write). -%%------------------------------------------------------------------- -ciphers_rsa_signed_certs() -> - [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_rsa_signed_certs(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:rsa_suites(crypto), - run_suites(Ciphers, Config, rsa). -%%------------------------------------------------------------------- -ciphers_rsa_signed_certs_openssl_names() -> - [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_rsa_suites(), - run_suites(Ciphers, Config, rsa). - -%%------------------------------------------------------------------- -ciphers_dsa_signed_certs() -> - [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_dsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:dsa_suites(NVersion), - run_suites(Ciphers, Config, dsa). -%%------------------------------------------------------------------- -ciphers_dsa_signed_certs_openssl_names() -> - [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_dsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_dsa_suites(), - run_suites(Ciphers, Config, dsa). - -%%------------------------------------------------------------------- -chacha_rsa_cipher_suites()-> - [{doc,"Test the cacha with ECDSA signed certs ciphersuites"}]. -chacha_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {KeyEx,_,_} = S <- ssl_test_lib:chacha_suites(NVersion), - KeyEx == ecdhe_rsa, KeyEx == dhe_rsa], - run_suites(Ciphers, Config, chacha_ecdsa). - -%%------------------------------------------------------------------- -chacha_ecdsa_cipher_suites()-> - [{doc,"Test the cacha with ECDSA signed certs ciphersuites"}]. -chacha_ecdsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:chacha_suites(NVersion)], - run_suites(Ciphers, Config, chacha_rsa). -%%----------------------------------------------------------------- -anonymous_cipher_suites()-> - [{doc,"Test the anonymous ciphersuites"}]. -anonymous_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdh_dh_anonymous_suites(NVersion), - run_suites(Ciphers, Config, anonymous). -%%------------------------------------------------------------------- -psk_cipher_suites() -> - [{doc, "Test the PSK ciphersuites WITHOUT server supplied identity hint"}]. -psk_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Config, psk). -%%------------------------------------------------------------------- -psk_with_hint_cipher_suites()-> - [{doc, "Test the PSK ciphersuites WITH server supplied identity hint"}]. -psk_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Config, psk_with_hint). -%%------------------------------------------------------------------- -psk_anon_cipher_suites() -> - [{doc, "Test the anonymous PSK ciphersuites WITHOUT server supplied identity hint"}]. -psk_anon_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Config, psk_anon). -%%------------------------------------------------------------------- -psk_anon_with_hint_cipher_suites()-> - [{doc, "Test the anonymous PSK ciphersuites WITH server supplied identity hint"}]. -psk_anon_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Config, psk_anon_with_hint). -%%------------------------------------------------------------------- -srp_cipher_suites()-> - [{doc, "Test the SRP ciphersuites"}]. -srp_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_suites(), - run_suites(Ciphers, Config, srp). -%%------------------------------------------------------------------- -srp_anon_cipher_suites()-> - [{doc, "Test the anonymous SRP ciphersuites"}]. -srp_anon_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_anon_suites(), - run_suites(Ciphers, Config, srp_anon). -%%------------------------------------------------------------------- -srp_dsa_cipher_suites()-> - [{doc, "Test the SRP DSA ciphersuites"}]. -srp_dsa_cipher_suites(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:srp_dss_suites(), - run_suites(Ciphers, Config, srp_dsa). -%%------------------------------------------------------------------- -rc4_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_rsa). -%------------------------------------------------------------------- -rc4_ecdh_rsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {ecdh_rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_ecdh_rsa). - -%%------------------------------------------------------------------- -rc4_ecdsa_cipher_suites()-> - [{doc, "Test the RC4 ciphersuites"}]. -rc4_ecdsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], - run_suites(Ciphers, Config, rc4_ecdsa). - -%%------------------------------------------------------------------- -des_rsa_cipher_suites()-> - [{doc, "Test the des_rsa ciphersuites"}]. -des_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:des_suites(NVersion)], - run_suites(Ciphers, Config, des_rsa). -%------------------------------------------------------------------- -des_ecdh_rsa_cipher_suites()-> - [{doc, "Test ECDH rsa signed ciphersuites"}]. -des_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = [S || {dhe_rsa,_,_} = S <- ssl_test_lib:des_suites(NVersion)], - run_suites(Ciphers, Config, des_dhe_rsa). - %%-------------------------------------------------------------------- default_reject_anonymous()-> [{doc,"Test that by default anonymous cipher suites are rejected "}]. @@ -2872,36 +2685,6 @@ default_reject_anonymous(Config) when is_list(Config) -> ssl_test_lib:check_server_alert(Server, Client, insufficient_security). %%-------------------------------------------------------------------- -ciphers_ecdsa_signed_certs() -> - [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdsa_suites(NVersion), - run_suites(Ciphers, Config, ecdsa). -%%-------------------------------------------------------------------- -ciphers_ecdsa_signed_certs_openssl_names() -> - [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_ecdsa_suites(), - run_suites(Ciphers, Config, ecdsa). -%%-------------------------------------------------------------------- -ciphers_ecdh_rsa_signed_certs() -> - [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdh_rsa_signed_certs(Config) when is_list(Config) -> - NVersion = ssl_test_lib:protocol_version(Config, tuple), - Ciphers = ssl_test_lib:ecdh_rsa_suites(NVersion), - run_suites(Ciphers, Config, ecdh_rsa). -%%-------------------------------------------------------------------- -ciphers_ecdh_rsa_signed_certs_openssl_names() -> - [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. - -ciphers_ecdh_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Ciphers = ssl_test_lib:openssl_ecdh_rsa_suites(), - run_suites(Ciphers, Config, ecdh_rsa). -%%-------------------------------------------------------------------- reuse_session() -> [{doc,"Test reuse of sessions (short handshake)"}]. reuse_session(Config) when is_list(Config) -> @@ -3463,9 +3246,9 @@ defaults(Config) when is_list(Config)-> true = lists:member(sslv3, proplists:get_value(available, Versions)), false = lists:member(sslv3, proplists:get_value(supported, Versions)), true = lists:member('tlsv1', proplists:get_value(available, Versions)), - true = lists:member('tlsv1', proplists:get_value(supported, Versions)), + false = lists:member('tlsv1', proplists:get_value(supported, Versions)), true = lists:member('tlsv1.1', proplists:get_value(available, Versions)), - true = lists:member('tlsv1.1', proplists:get_value(supported, Versions)), + false = lists:member('tlsv1.1', proplists:get_value(supported, Versions)), true = lists:member('tlsv1.2', proplists:get_value(available, Versions)), true = lists:member('tlsv1.2', proplists:get_value(supported, Versions)), false = lists:member({rsa,rc4_128,sha}, ssl:cipher_suites()), @@ -3477,7 +3260,7 @@ defaults(Config) when is_list(Config)-> true = lists:member('dtlsv1.2', proplists:get_value(available_dtls, Versions)), true = lists:member('dtlsv1', proplists:get_value(available_dtls, Versions)), true = lists:member('dtlsv1.2', proplists:get_value(supported_dtls, Versions)), - true = lists:member('dtlsv1', proplists:get_value(supported_dtls, Versions)). + false = lists:member('dtlsv1', proplists:get_value(supported_dtls, Versions)). %%-------------------------------------------------------------------- reuseaddr() -> @@ -5849,6 +5632,29 @@ tls13_unsupported_sign_algo_cert_client_auth_ssl_server_openssl_client(Config) - ssl_test_lib:close_port(Client). +tls13_connection_information() -> + [{doc,"Test the API function ssl:connection_information/1 in a TLS 1.3 connection"}]. + +tls13_connection_information(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), + %% Set versions + ServerOpts = [{versions, ['tlsv1.2','tlsv1.3']}|ServerOpts0], + {_ClientNode, ServerNode, _Hostname} = ssl_test_lib:run_where(Config), + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, connection_information_result, []}}, + {options, ServerOpts}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_basic_client(openssl, 'tlsv1.3', Port, ClientOpts), + + ssl_test_lib:check_result(Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close_port(Client). + + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -6332,147 +6138,6 @@ client_server_opts(#{key_exchange := KeyAlgo}, Config) when KeyAlgo == ecdh_rsa {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)}. -run_suites(Ciphers, Config, Type) -> - Version = ssl_test_lib:protocol_version(Config), - ct:log("Running cipher suites ~p~n", [Ciphers]), - {ClientOpts, ServerOpts} = - case Type of - rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_opts, Config)]}; - dsa -> - {ssl_test_lib:ssl_options(client_dsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_dsa_opts, Config)]}; - anonymous -> - %% No certs in opts! - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options([], Config)]}; - psk -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk, Config)]}; - psk_with_hint -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_hint, Config) - ]}; - psk_anon -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_anon, Config)]}; - psk_anon_with_hint -> - {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_psk_anon_hint, Config)]}; - srp -> - {ssl_test_lib:ssl_options(client_srp, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp, Config)]}; - srp_anon -> - {ssl_test_lib:ssl_options(client_srp, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp_anon, Config)]}; - srp_dsa -> - {ssl_test_lib:ssl_options(client_srp_dsa, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_srp_dsa, Config)]}; - ecdsa -> - {ssl_test_lib:ssl_options(client_ecdsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]}; - ecdh_rsa -> - {ssl_test_lib:ssl_options(client_ecdh_rsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)]}; - rc4_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - rc4_ecdh_rsa -> - {ssl_test_lib:ssl_options(client_ecdh_rsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)]}; - rc4_ecdsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]}; - des_dhe_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_verification_opts, Config)]}; - des_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - chacha_rsa -> - {ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_rsa_verify_opts, Config)]}; - chacha_ecdsa -> - {ssl_test_lib:ssl_options(client_ecdsa_opts, Config), - [{ciphers, Ciphers} | - ssl_test_lib:ssl_options(server_ecdsa_opts, Config)]} - end, - Suites = ssl_test_lib:filter_suites(Ciphers, Version), - ct:pal("ssl_test_lib:filter_suites(~p ~p) -> ~p ", [Ciphers, Version, Suites]), - Results0 = lists:map(fun(Cipher) -> - cipher(Cipher, Version, Config, ClientOpts, ServerOpts) end, - ssl_test_lib:filter_suites(Ciphers, Version)), - Results = lists:flatten(Results0), - true = length(Results) == length(Suites), - check_cipher_result(Results). - -check_cipher_result([]) -> - ok; -check_cipher_result([ok | Rest]) -> - check_cipher_result(Rest); -check_cipher_result([_ |_] = Error) -> - ct:fail(Error). - -erlang_cipher_suite(Suite) when is_list(Suite)-> - ssl_cipher_format:suite_definition(ssl_cipher_format:openssl_suite(Suite)); -erlang_cipher_suite(Suite) -> - Suite. - -cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> - %% process_flag(trap_exit, true), - ct:log("Testing CipherSuite ~p~n", [CipherSuite]), - ct:log("Server Opts ~p~n", [ServerOpts]), - ct:log("Client Opts ~p~n", [ClientOpts]), - {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - - ErlangCipherSuite = erlang_cipher_suite(CipherSuite), - - ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, - - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, ServerOpts}]), - Port = ssl_test_lib:inet_port(Server), - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, - {options, - [{ciphers,[CipherSuite]} | - ClientOpts]}]), - - Result = ssl_test_lib:wait_for_result(Server, ok, Client, ok), - - ssl_test_lib:close(Server), - ssl_test_lib:close(Client), - - case Result of - ok -> - [ok]; - Error -> - [{ErlangCipherSuite, Error}] - end. - connection_information_result(Socket) -> {ok, Info = [_ | _]} = ssl:connection_information(Socket), case length(Info) > 3 of diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl index 8690faed54..4f340af4f5 100644 --- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl +++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl @@ -147,6 +147,7 @@ init_per_testcase(_TestCase, Config) -> ssl:stop(), ssl:start(), ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:pal(" ~p", [ dtls_record:supported_protocol_versions()]), ct:timetrap({seconds, 10}), Config. diff --git a/lib/ssl/test/ssl_cipher_suite_SUITE.erl b/lib/ssl/test/ssl_cipher_suite_SUITE.erl new file mode 100644 index 0000000000..bf1bc0e752 --- /dev/null +++ b/lib/ssl/test/ssl_cipher_suite_SUITE.erl @@ -0,0 +1,772 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-module(ssl_cipher_suite_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +all() -> + [ + {group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + [ + {'tlsv1.2', [], kex()}, + {'tlsv1.1', [], kex()}, + {'tlsv1', [], kex()}, + {'sslv3', [], kex()}, + {'dtlsv1.2', [], kex()}, + {'dtlsv1', [], kex()}, + {dhe_rsa, [],[dhe_rsa_3des_ede_cbc, + dhe_rsa_aes_128_cbc, + dhe_rsa_aes_256_cbc, + dhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_rsa, [], [ecdhe_rsa_3des_ede_cbc, + ecdhe_rsa_aes_128_cbc, + ecdhe_rsa_aes_128_gcm, + ecdhe_rsa_aes_256_cbc, + ecdhe_rsa_aes_256_gcm, + ecdhe_rsa_chacha20_poly1305 + ]}, + {ecdhe_ecdsa, [],[ecdhe_ecdsa_rc4_128, + ecdhe_ecdsa_3des_ede_cbc, + ecdhe_ecdsa_aes_128_cbc, + ecdhe_ecdsa_aes_128_gcm, + ecdhe_ecdsa_aes_256_cbc, + ecdhe_ecdsa_aes_256_gcm, + ecdhe_ecdsa_chacha20_poly1305 + ]}, + {rsa, [], [rsa_3des_ede_cbc, + rsa_aes_128_cbc, + rsa_aes_256_cbc, + rsa_rc4_128 + ]}, + {dhe_dss, [], [dhe_dss_3des_ede_cbc, + dhe_dss_aes_128_cbc, + dhe_dss_aes_256_cbc]}, + {srp_rsa, [], [srp_rsa_3des_ede_cbc, + srp_rsa_aes_128_cbc, + srp_rsa_aes_256_cbc]}, + {srp_dss, [], [srp_dss_3des_ede_cbc, + srp_dss_aes_128_cbc, + srp_dss_aes_256_cbc]}, + {rsa_psk, [], [rsa_psk_3des_ede_cbc, + rsa_psk_rc4_128, + rsa_psk_aes_128_cbc, + rsa_psk_aes_256_cbc + ]}, + {dh_anon, [], [dh_anon_rc4_128, + dh_anon_3des_ede_cbc, + dh_anon_aes_128_cbc, + dh_anon_aes_128_gcm, + dh_anon_aes_256_cbc, + dh_anon_aes_256_gcm]}, + {ecdh_anon, [], [ecdh_anon_3des_ede_cbc, + ecdh_anon_aes_128_cbc, + ecdh_anon_aes_256_cbc + ]}, + {srp_anon, [], [srp_anon_3des_ede_cbc, + srp_anon_aes_128_cbc, + srp_anon_aes_256_cbc]}, + {psk, [], [psk_3des_ede_cbc, + psk_rc4_128, + psk_aes_128_cbc, + psk_aes_128_ccm, + psk_aes_128_ccm_8, + psk_aes_256_cbc, + psk_aes_256_ccm, + psk_aes_256_ccm_8 + ]}, + {dhe_psk, [], [dhe_psk_3des_ede_cbc, + dhe_psk_rc4_128, + dhe_psk_aes_128_cbc, + dhe_psk_aes_128_ccm, + dhe_psk_aes_128_ccm_8, + dhe_psk_aes_256_cbc, + dhe_psk_aes_256_ccm, + dhe_psk_aes_256_ccm_8 + ]}, + {ecdhe_psk, [], [ecdhe_psk_3des_ede_cbc, + ecdhe_psk_rc4_128, + ecdhe_psk_aes_128_cbc, + ecdhe_psk_aes_128_ccm, + ecdhe_psk_aes_128_ccm_8, + ecdhe_psk_aes_256_cbc + ]} + ]. + + +kex() -> + rsa() ++ ecdsa() ++ dss() ++ anonymous(). + +rsa() -> + [{group, dhe_rsa}, + {group, ecdhe_rsa}, + {group, rsa}, + {group, srp_rsa}, + {group, rsa_psk} + ]. + +ecdsa() -> + [{group, ecdhe_ecdsa}]. + +dss() -> + [{group, dhe_dss}, + {group, srp_dss}]. + +anonymous() -> + [{group, dh_anon}, + {group, ecdh_anon}, + {group, psk}, + {group, dhe_psk}, + {group, ecdhe_psk}, + {group, srp_anon} + ]. + + +init_per_suite(Config) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl_test_lib:clean_start(), + Config + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_Config) -> + ssl:stop(), + application:stop(crypto). + +%%-------------------------------------------------------------------- +init_per_group(GroupName, Config) when GroupName == ecdh_anon; + GroupName == ecdhe_rsa; + GroupName == ecdhe_psk -> + case proplists:get_bool(ecdh, proplists:get_value(public_keys, crypto:supports())) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +init_per_group(ecdhe_ecdsa = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(ecdh, PKAlg) andalso lists:member(ecdsa, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing EC crypto support"} + end; +init_per_group(dhe_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS crypto support"} + end; +init_per_group(srp_dss = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dss, PKAlg) andalso lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing DSS_SRP crypto support"} + end; +init_per_group(GroupName, Config) when GroupName == srp_anon; + GroupName == srp_rsa -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(srp, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +init_per_group(dhe_psk = GroupName, Config) -> + PKAlg = proplists:get_value(public_keys, crypto:supports()), + case lists:member(dh, PKAlg) of + true -> + init_certs(GroupName, Config); + false -> + {skip, "Missing SRP crypto support"} + end; +init_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:init_tls_version(GroupName, end_per_group(GroupName, Config0)); + false -> + init_certs(GroupName, Config0) + end. + +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. +init_per_testcase(TestCase, Config) when TestCase == psk_3des_ede_cbc; + TestCase == srp_anon_3des_ede_cbc; + TestCase == dhe_psk_3des_ede_cbc; + TestCase == ecdhe_psk_3des_ede_cbc; + TestCase == srp_rsa_3des_ede_cbc; + TestCase == srp_dss_3des_ede_cbc; + TestCase == rsa_psk_3des_ede_cbc; + TestCase == rsa_3des_ede_cbc; + TestCase == dhe_rsa_3des_ede_cbc; + TestCase == dhe_dss_3des_ede_cbc; + TestCase == ecdhe_rsa_3des_ede_cbc; + TestCase == srp_anon_dss_3des_ede_cbc; + TestCase == dh_anon_3des_ede_cbc; + TestCase == ecdh_anon_3des_ede_cbc; + TestCase == ecdhe_ecdsa_3des_ede_cbc -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(des_ede3, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing 3DES crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_rc4_128; + TestCase == ecdhe_psk_rc4_128; + TestCase == dhe_psk_rc4_128; + TestCase == rsa_psk_rc4_128; + TestCase == rsa_rc4_128; + TestCase == ecdhe_rsa_rc4_128; + TestCase == ecdhe_ecdsa_rc4_128; + TestCase == dh_anon_rc4_128 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(rc4, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing RC4 crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_128_ccm_8; + TestCase == rsa_psk_aes_128_ccm_8; + TestCase == psk_aes_128_ccm_8; + TestCase == dhe_psk_aes_128_ccm_8; + TestCase == ecdhe_psk_aes_128_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_128_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_128_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) when TestCase == psk_aes_256_ccm_8; + TestCase == rsa_psk_aes_256_ccm_8; + TestCase == psk_aes_256_ccm_8; + TestCase == dhe_psk_aes_256_ccm_8; + TestCase == ecdhe_psk_aes_256_ccm_8 -> + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(aes_256_ccm, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, "Missing AES_256_CCM crypto support"} + end; +init_per_testcase(TestCase, Config) -> + Cipher = test_cipher(TestCase, Config), + %%Reason = io_lib:format("Missing ~p crypto support", [Cipher]), + SupCiphers = proplists:get_value(ciphers, crypto:supports()), + case lists:member(Cipher, SupCiphers) of + true -> + ct:timetrap({seconds, 5}), + Config; + _ -> + {skip, {Cipher, SupCiphers}} + end. + +end_per_testcase(_TestCase, Config) -> + Config. + +init_certs(srp_rsa, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_anon, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa_psk, Config) -> + ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[],[],[{extensions, ClientExt}]]}], + Config, "_peer_keyEncipherment"), + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, PskSharedSecret}} | ServerOpts], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(rsa, Config) -> + ClientExt = x509_test:extensions([{key_usage, [digitalSignature, keyEncipherment]}]), + {ClientOpts, ServerOpts} = ssl_test_lib:make_rsa_cert_chains([{server_chain, + [[],[],[{extensions, ClientExt}]]}], + Config, "_peer_keyEncipherment"), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(dhe_dss, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(dsa, dsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(srp_dss, Config) -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(dsa, dsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}} | ServerOpts], + client_config => [{srp_identity, {"Test-User", "secret"}} | ClientOpts]}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_rsa; + GroupName == ecdhe_rsa -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(rsa, rsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == dhe_ecdsa; + GroupName == ecdhe_ecdsa -> + DefConf = ssl_test_lib:default_cert_chain_conf(), + CertChainConf = ssl_test_lib:gen_conf(ecdsa, ecdsa, DefConf, DefConf), + #{server_config := ServerOpts, + client_config := ClientOpts} + = public_key:pkix_test_data(CertChainConf), + [{tls_config, #{server_config => ServerOpts, + client_config => ClientOpts}} | + proplists:delete(tls_config, Config)]; +init_certs(GroupName, Config) when GroupName == psk; + GroupName == dhe_psk; + GroupName == ecdhe_psk -> + PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}], + client_config => [{psk_identity, "Test-User"}, + {user_lookup_fun, {fun user_lookup/3, PskSharedSecret}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(srp, Config) -> + [{tls_config, #{server_config => [{user_lookup_fun, {fun user_lookup/3, undefined}}], + client_config => [{srp_identity, {"Test-User", "secret"}}]}} | + proplists:delete(tls_config, Config)]; +init_certs(_GroupName, Config) -> + %% Anonymous does not need certs + [{tls_config, #{server_config => [], + client_config => []}} | + proplists:delete(tls_config, Config)]. +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% SRP -------------------------------------------------------- +%%-------------------------------------------------------------------- +srp_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, '3des_ede_cbc', Config). + +srp_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_128_cbc', Config). + +srp_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_rsa, 'aes_256_cbc', Config). + +srp_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, '3des_ede_cbc', Config). + +srp_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_128_cbc', Config). + +srp_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_dss, 'aes_256_cbc', Config). + +%%-------------------------------------------------------------------- +%% PSK -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, '3des_ede_cbc', Config). + +rsa_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_cbc', Config). + +rsa_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm', Config). + +rsa_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_128_ccm_8', Config). + +rsa_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_cbc', Config). + +rsa_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm', Config). + +rsa_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'aes_256_ccm_8', Config). + +rsa_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa_psk, 'rc4_128', Config). + +%%-------------------------------------------------------------------- +%% RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +rsa_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'des_cbc', Config). + +rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, '3des_ede_cbc', Config). + +rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_cbc', Config). + +rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_cbc', Config). + +rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_128_gcm', Config). + +rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'aes_256_gcm', Config). + +rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(rsa, 'rc4_128', Config). +%%-------------------------------------------------------------------- +%% DHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, '3des_ede_cbc', Config). + +dhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_cbc', Config). + +dhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_128_gcm', Config). + +dhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_cbc', Config). + +dhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'aes_256_gcm', Config). + +dhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(dhe_rsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% ECDHE_RSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_rsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, '3des_ede_cbc', Config). + +ecdhe_rsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_cbc', Config). + +ecdhe_rsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_128_gcm', Config). + +ecdhe_rsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_cbc', Config). + +ecdhe_rsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'aes_256_gcm', Config). + +ecdhe_rsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'rc4_128', Config). + +ecdhe_rsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_rsa, 'chacha20_poly1305', Config). + +%%-------------------------------------------------------------------- +%% ECDHE_ECDSA -------------------------------------------------------- +%%-------------------------------------------------------------------- +ecdhe_ecdsa_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'rc4_128', Config). + +ecdhe_ecdsa_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, '3des_ede_cbc', Config). + +ecdhe_ecdsa_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_cbc', Config). + +ecdhe_ecdsa_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_128_gcm', Config). + +ecdhe_ecdsa_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_cbc', Config). + +ecdhe_ecdsa_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'aes_256_gcm', Config). + +ecdhe_ecdsa_chacha20_poly1305(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_ecdsa, 'chacha20_poly1305', Config). +%%-------------------------------------------------------------------- +%% DHE_DSS -------------------------------------------------------- +%%-------------------------------------------------------------------- +dhe_dss_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'des_cbc', Config). + +dhe_dss_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, '3des_ede_cbc', Config). + +dhe_dss_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_cbc', Config). + +dhe_dss_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_cbc', Config). + +dhe_dss_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_128_gcm', Config). + +dhe_dss_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_dss, 'aes_256_gcm', Config). + +%%-------------------------------------------------------------------- +%% Anonymous -------------------------------------------------------- +%%-------------------------------------------------------------------- +dh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, '3des_ede_cbc', Config). + +dh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_cbc', Config). + +dh_anon_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_128_gcm', Config). + +dh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_cbc', Config). + +dh_anon_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'aes_256_gcm', Config). + +dh_anon_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dh_anon, 'rc4_128', Config). + +ecdh_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, '3des_ede_cbc', Config). + +ecdh_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_128_cbc', Config). + +ecdh_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdh_anon, 'aes_256_cbc', Config). + +srp_anon_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, '3des_ede_cbc', Config). + +srp_anon_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_128_cbc', Config). + +srp_anon_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(srp_anon, 'aes_256_cbc', Config). + +dhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'des_cbc', Config). + +dhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'rc4_128', Config). + +dhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, '3des_ede_cbc', Config). + +dhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_cbc', Config). + +dhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_cbc', Config). + +dhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_gcm', Config). + +dhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_gcm', Config). + +dhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm', Config). + +dhe_psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm', Config). + +dhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_128_ccm_8', Config). + +dhe_psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(dhe_psk, 'aes_256_ccm_8', Config). + +ecdhe_psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'des_cbc', Config). + +ecdhe_psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'rc4_128', Config). + +ecdhe_psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, '3des_ede_cbc', Config). + +ecdhe_psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_cbc', Config). + +ecdhe_psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_cbc', Config). + +ecdhe_psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_gcm', Config). + +ecdhe_psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_256_gcm', Config). + +ecdhe_psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm', Config). + +ecdhe_psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(ecdhe_psk, 'aes_128_ccm_8', Config). + +psk_des_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'des_cbc', Config). + +psk_rc4_128(Config) when is_list(Config) -> + run_ciphers_test(psk, 'rc4_128', Config). + +psk_3des_ede_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, '3des_ede_cbc', Config). + +psk_aes_128_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_cbc', Config). + +psk_aes_256_cbc(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_cbc', Config). + +psk_aes_128_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_gcm', Config). + +psk_aes_256_gcm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_gcm', Config). + +psk_aes_128_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm', Config). + +psk_aes_256_ccm(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm', Config). + +psk_aes_128_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_128_ccm_8', Config). + +psk_aes_256_ccm_8(Config) when is_list(Config) -> + run_ciphers_test(psk, 'aes_256_ccm_8', Config). + +%%-------------------------------------------------------------------- +%% Internal functions ---------------------------------------------- +%%-------------------------------------------------------------------- +test_cipher(TestCase, Config) -> + [{name, Group} |_] = proplists:get_value(tc_group_properties, Config), + list_to_atom(re:replace(atom_to_list(TestCase), atom_to_list(Group) ++ "_", "", [{return, list}])). + +run_ciphers_test(Kex, Cipher, Config) -> + Version = ssl_test_lib:protocol_version(Config), + TestCiphers = test_ciphers(Kex, Cipher, Version), + + case TestCiphers of + [_|_] -> + lists:foreach(fun(TestCipher) -> + cipher_suite_test(TestCipher, Version, Config) + end, TestCiphers); + [] -> + {skip, {not_sup, Kex, Cipher, Version}} + end. + +cipher_suite_test(CipherSuite, Version, Config) -> + #{server_config := SOpts, + client_config := COpts} = proplists:get_value(tls_config, Config), + ServerOpts = ssl_test_lib:ssl_options(SOpts, Config), + ClientOpts = ssl_test_lib:ssl_options(COpts, Config), + ct:log("Testing CipherSuite ~p~n", [CipherSuite]), + ct:log("Server Opts ~p~n", [ServerOpts]), + ct:log("Client Opts ~p~n", [ClientOpts]), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + ErlangCipherSuite = erlang_cipher_suite(CipherSuite), + + ConnectionInfo = {ok, {Version, ErlangCipherSuite}}, + + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, + {options, [{versions, [Version]}, {ciphers, [CipherSuite]} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {ssl_test_lib, cipher_result, [ConnectionInfo]}}, + {options, [{versions, [Version]}, {ciphers, [CipherSuite]} | + ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +erlang_cipher_suite(Suite) when is_list(Suite)-> + ssl_cipher_format:suite_definition(ssl_cipher_format:openssl_suite(Suite)); +erlang_cipher_suite(Suite) -> + Suite. + +user_lookup(psk, _Identity, UserState) -> + {ok, UserState}; +user_lookup(srp, Username, _UserState) -> + Salt = ssl_cipher:random_bytes(16), + UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), + {ok, {srp_1024, Salt, UserPassHash}}. + +test_ciphers(Kex, Cipher, Version) -> + ssl:filter_cipher_suites(ssl:cipher_suites(all, Version) ++ ssl:cipher_suites(anonymous, Version), + [{key_exchange, + fun(Kex0) when Kex0 == Kex -> true; + (_) -> false + end}, + {cipher, + fun(Cipher0) when Cipher0 == Cipher -> true; + (_) -> false + end}]). diff --git a/lib/ssl/test/ssl_dist_bench_SUITE.erl b/lib/ssl/test/ssl_dist_bench_SUITE.erl index 7e7de5c9bf..1fea6f6f72 100644 --- a/lib/ssl/test/ssl_dist_bench_SUITE.erl +++ b/lib/ssl/test/ssl_dist_bench_SUITE.erl @@ -49,10 +49,14 @@ suite() -> [{ct_hooks, [{ts_install_cth, [{nodenames, 2}]}]}]. -all() -> [{group, ssl}, {group, plain}]. +all() -> + [{group, ssl}, + {group, crypto}, + {group, plain}]. groups() -> [{ssl, all_groups()}, + {crypto, all_groups()}, {plain, all_groups()}, %% {setup, [{repeat, 1}], [setup]}, @@ -164,6 +168,17 @@ end_per_suite(Config) -> init_per_group(ssl, Config) -> [{ssl_dist, true}, {ssl_dist_prefix, "SSL"}|Config]; +init_per_group(crypto, Config) -> + case inet_crypto_dist:is_supported() of + true -> + [{ssl_dist, false}, {ssl_dist_prefix, "Crypto"}, + {ssl_dist_args, + "-proto_dist inet_crypto " + "-inet_crypto '#{secret => \"123456\"}'"} + |Config]; + false -> + {skip, "Not supported on this OTP version"} + end; init_per_group(plain, Config) -> [{ssl_dist, false}, {ssl_dist_prefix, "Plain"}|Config]; init_per_group(_GroupName, Config) -> @@ -374,29 +389,46 @@ sched_utilization(A, B, Prefix, HA, HB, SSL) -> [A] = ssl_apply(HB, erlang, nodes, []), msacc:print(ClientMsacc), msacc:print(ServerMsacc), - ct:pal("Got ~p msgs",[length(Msgs)]), - report(Prefix++" Sched Utilization Client", - 10000 * msacc:stats(system_runtime,ClientMsacc) / - msacc:stats(system_realtime,ClientMsacc), "util 0.01 %"), - report(Prefix++" Sched Utilization Server", - 10000 * msacc:stats(system_runtime,ServerMsacc) / - msacc:stats(system_realtime,ServerMsacc), "util 0.01 %"), - ok. + ct:pal("Got ~p busy_dist_port msgs",[length(Msgs)]), + ct:log("Stats of B from A: ~p", + [ssl_apply(HA, net_kernel, node_info, [B])]), + ct:log("Stats of A from B: ~p", + [ssl_apply(HB, net_kernel, node_info, [A])]), + SchedUtilClient = + round(10000 * msacc:stats(system_runtime,ClientMsacc) / + msacc:stats(system_realtime,ClientMsacc)), + SchedUtilServer = + round(10000 * msacc:stats(system_runtime,ServerMsacc) / + msacc:stats(system_realtime,ServerMsacc)), + Verdict = + case Msgs of + [] -> + ""; + _ -> + " ???" + end, + {comment, ClientComment} = + report(Prefix ++ " Sched Utilization Client" ++ Verdict, + SchedUtilClient, "/100 %" ++ Verdict), + {comment, ServerComment} = + report(Prefix++" Sched Utilization Server" ++ Verdict, + SchedUtilServer, "/100 %" ++ Verdict), + {comment, "Client " ++ ClientComment ++ ", Server " ++ ServerComment}. %% Runs on node A and spawns a server on node B %% We want to avoid getting busy_dist_port as it hides the true SU usage %% of the receiver and sender. sched_util_runner(A, B, true) -> - sched_util_runner(A, B, 50); + sched_util_runner(A, B, 250); sched_util_runner(A, B, false) -> sched_util_runner(A, B, 250); sched_util_runner(A, B, Senders) -> Payload = payload(5), [A] = rpc:call(B, erlang, nodes, []), - ServerPid = - erlang:spawn( - B, - fun () -> throughput_server() end), + ServerPids = + [erlang:spawn_link( + B, fun () -> throughput_server() end) + || _ <- lists:seq(1, Senders)], ServerMsacc = erlang:spawn( B, @@ -404,24 +436,28 @@ sched_util_runner(A, B, Senders) -> receive {start,Pid} -> msacc:start(10000), - Pid ! {ServerPid,msacc:stats()} + receive + {done,Pid} -> + Pid ! {self(),msacc:stats()} + end end end), - spawn_link( - fun() -> - %% We spawn 250 senders which should mean that we - %% have a load of 250 msgs/msec - [spawn_link( - fun() -> - throughput_client(ServerPid,Payload) - end) || _ <- lists:seq(1, Senders)] - end), - erlang:system_monitor(self(),[busy_dist_port]), + %% We spawn 250 senders which should mean that we + %% have a load of 250 msgs/msec + [spawn_link( + fun() -> + throughput_client(Pid, Payload) + end) || Pid <- ServerPids], + %% + receive after 1000 -> ok end, ServerMsacc ! {start,self()}, msacc:start(10000), ClientMsaccStats = msacc:stats(), - ServerMsaccStats = receive {ServerPid,Stats} -> Stats end, + receive after 1000 -> ok end, + ServerMsacc ! {done,self()}, + ServerMsaccStats = receive {ServerMsacc,Stats} -> Stats end, + %% {ClientMsaccStats,ServerMsaccStats, flush()}. flush() -> @@ -522,15 +558,20 @@ throughput(A, B, Prefix, HA, HB, Packets, Size) -> + byte_size(erlang:term_to_binary([0|<<>>])), % Benchmark overhead Bytes = Packets * (Size + Overhead), io:format("~w bytes, ~.4g s~n", [Bytes,Time/1000000]), + SizeString = integer_to_list(Size), ClientMsaccStats =:= undefined orelse - io:format( - "Sender core usage ratio: ~.4g ns/byte~n", - [msacc:stats(system_runtime, ClientMsaccStats)*1000/Bytes]), + report( + Prefix ++ " Sender_RelativeCoreLoad_" ++ SizeString, + round(msacc:stats(system_runtime, ClientMsaccStats) + * 1000000 / Bytes), + "ps/byte"), ServerMsaccStats =:= undefined orelse begin - io:format( - "Receiver core usage ratio: ~.4g ns/byte~n", - [msacc:stats(system_runtime, ServerMsaccStats)*1000/Bytes]), + report( + Prefix ++ " Receiver_RelativeCoreLoad_" ++ SizeString, + round(msacc:stats(system_runtime, ServerMsaccStats) + * 1000000 / Bytes), + "ps/byte"), msacc:print(ServerMsaccStats) end, io:format("******* ClientProf:~n", []), prof_print(ClientProf), @@ -538,7 +579,7 @@ throughput(A, B, Prefix, HA, HB, Packets, Size) -> io:format("******* Server GC Before:~n~p~n", [Server_GC_Before]), io:format("******* Server GC After:~n~p~n", [Server_GC_After]), Speed = round((Bytes * 1000000) / (1024 * Time)), - report(Prefix++" Throughput_"++integer_to_list(Size), Speed, "kB/s"). + report(Prefix ++ " Throughput_" ++ SizeString, Speed, "kB/s"). %% Runs on node A and spawns a server on node B throughput_runner(A, B, Rounds, Size) -> @@ -546,11 +587,12 @@ throughput_runner(A, B, Rounds, Size) -> [A] = rpc:call(B, erlang, nodes, []), ClientPid = self(), ServerPid = - erlang:spawn( + erlang:spawn_opt( B, - fun () -> throughput_server(ClientPid, Rounds) end), + fun () -> throughput_server(ClientPid, Rounds) end, + [{message_queue_data, off_heap}]), ServerMon = erlang:monitor(process, ServerPid), - msacc:available() andalso + msacc_available() andalso begin msacc:stop(), msacc:reset(), @@ -562,7 +604,7 @@ throughput_runner(A, B, Rounds, Size) -> throughput_client(ServerPid, ServerMon, Payload, Rounds), prof_stop(), MsaccStats = - case msacc:available() of + case msacc_available() of true -> MStats = msacc:stats(), msacc:stop(), @@ -602,7 +644,7 @@ throughput_server(Pid, N) -> GC_Before = get_server_gc_info(), %% dbg:tracer(port, dbg:trace_port(file, "throughput_server_gc.log")), %% dbg:p(TLSDistReceiver, garbage_collection), - msacc:available() andalso + msacc_available() andalso begin msacc:stop(), msacc:reset(), @@ -615,7 +657,7 @@ throughput_server(Pid, N) -> throughput_server_loop(_Pid, GC_Before, 0) -> prof_stop(), MsaccStats = - case msacc:available() of + case msacc_available() of true -> msacc:stop(), MStats = msacc:stats(), @@ -632,8 +674,13 @@ throughput_server_loop(_Pid, GC_Before, 0) -> server_gc_after => get_server_gc_info()}); throughput_server_loop(Pid, GC_Before, N) -> receive - {Pid, N, _} -> - throughput_server_loop(Pid, GC_Before, N-1) + Msg -> + case Msg of + {Pid, N, _} -> + throughput_server_loop(Pid, GC_Before, N - 1); + Other -> + erlang:error({self(),?FUNCTION_NAME,Other}) + end end. get_server_gc_info() -> @@ -773,7 +820,7 @@ get_node_args(Tag, Config) -> true -> proplists:get_value(Tag, Config); false -> - "" + proplists:get_value(ssl_dist_args, Config, "") end. @@ -828,3 +875,6 @@ report(Name, Value, Unit) -> term_to_string(Term) -> unicode:characters_to_list( io_lib:write(Term, [{encoding, unicode}])). + +msacc_available() -> + msacc:available(). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 7f8e81dbd8..22169035f3 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -514,13 +514,6 @@ wait_for_result(Pid, Msg) -> %% Unexpected end. -user_lookup(psk, _Identity, UserState) -> - {ok, UserState}; -user_lookup(srp, Username, _UserState) -> - Salt = ssl_cipher:random_bytes(16), - UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), - {ok, {srp_1024, Salt, UserPassHash}}. - cert_options(Config) -> ClientCaCertFile = filename:join([proplists:get_value(priv_dir, Config), "client", "cacerts.pem"]), @@ -1540,10 +1533,13 @@ cipher_result(Socket, Result) -> ct:log("~p:~p~nSuccessfull connect: ~p~n", [?MODULE,?LINE, Result]), %% Importante to send two packets here %% to properly test "cipher state" handling - ssl:send(Socket, "Hello\n"), - "Hello\n" = active_recv(Socket, length( "Hello\n")), - ssl:send(Socket, " world\n"), - " world\n" = active_recv(Socket, length(" world\n")), + Hello = "Hello\n", + World = " world\n", + ssl:send(Socket, Hello), + ct:sleep(500), + ssl:send(Socket, World), + Expected = Hello ++ World, + Expected = active_recv(Socket, length(Expected)), ok. session_info_result(Socket) -> @@ -2378,4 +2374,10 @@ reuse_session(ClientOpts, ServerOpts, Config) -> ssl_test_lib:close(Server1), ssl_test_lib:close(Client3), ssl_test_lib:close(Client4). - + +user_lookup(psk, _Identity, UserState) -> + {ok, UserState}; +user_lookup(srp, Username, _UserState) -> + Salt = ssl_cipher:random_bytes(16), + UserPassHash = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, <<"secret">>])]), + {ok, {srp_1024, Salt, UserPassHash}}. diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index df84411b6d..f22eb4ecdf 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -39,16 +39,14 @@ all() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{group, basic}, - {group, 'tlsv1.2'}, + [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, {group, 'sslv3'}, {group, 'dtlsv1.2'}, {group, 'dtlsv1'}]; false -> - [{group, basic}, - {group, 'tlsv1.2'}, + [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, {group, 'sslv3'}] @@ -57,8 +55,7 @@ all() -> groups() -> case ssl_test_lib:openssl_sane_dtls() of true -> - [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + [{'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()}, @@ -66,20 +63,13 @@ groups() -> {'dtlsv1', [], dtls_all_versions_tests()} ]; false -> - [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + [{'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()} ] end. - -basic_tests() -> - [basic_erlang_client_openssl_server, - basic_erlang_server_openssl_client, - expired_session - ]. - + all_versions_tests() -> [ erlang_client_openssl_server, @@ -357,85 +347,7 @@ end_per_testcase(_, Config) -> %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- -basic_erlang_client_openssl_server() -> - [{doc,"Test erlang client with openssl server"}]. -basic_erlang_client_openssl_server(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), - - {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), - - Data = "From openssl to erlang", - - Port = ssl_test_lib:inet_port(node()), - CertFile = proplists:get_value(certfile, ServerOpts), - KeyFile = proplists:get_value(keyfile, ServerOpts), - - Exe = "openssl", - Args = ["s_server", "-accept", integer_to_list(Port), - "-cert", CertFile, "-key", KeyFile], - - OpensslPort = ssl_test_lib:portable_open_port(Exe, Args), - - - ssl_test_lib:wait_for_openssl_server(Port, tls), - - Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, - {host, Hostname}, - {from, self()}, - {mfa, {?MODULE, - erlang_ssl_receive, [Data]}}, - {options, ClientOpts}]), - true = port_command(OpensslPort, Data), - - ssl_test_lib:check_result(Client, ok), - - %% Clean close down! Server needs to be closed first !! - ssl_test_lib:close_port(OpensslPort), - ssl_test_lib:close(Client), - process_flag(trap_exit, false). - -%%-------------------------------------------------------------------- -basic_erlang_server_openssl_client() -> - [{doc,"Test erlang server with openssl client"}]. -basic_erlang_server_openssl_client(Config) when is_list(Config) -> - process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - - {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), - - Data = "From openssl to erlang", - Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, - {from, self()}, - {mfa, {?MODULE, erlang_ssl_receive, [Data]}}, - {options,ServerOpts}]), - - Port = ssl_test_lib:inet_port(Server), - - Exe = "openssl", - Args = case no_low_flag("-no_ssl2") of - [] -> - ["s_client", "-connect", hostname_format(Hostname) ++ - ":" ++ integer_to_list(Port), no_low_flag("-no_ssl3") - | workaround_openssl_s_clinent()]; - Flag -> - ["s_client", "-connect", hostname_format(Hostname) ++ - ":" ++ integer_to_list(Port), no_low_flag("-no_ssl3"), Flag - | workaround_openssl_s_clinent()] - end, - - OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), - true = port_command(OpenSslPort, Data), - - ssl_test_lib:check_result(Server, ok), - - %% Clean close down! Server needs to be closed first !! - ssl_test_lib:close(Server), - ssl_test_lib:close_port(OpenSslPort), - process_flag(trap_exit, false). -%%-------------------------------------------------------------------- erlang_client_openssl_server() -> [{doc,"Test erlang client with openssl server"}]. erlang_client_openssl_server(Config) when is_list(Config) -> diff --git a/lib/ssl/test/ssl_upgrade_SUITE.erl b/lib/ssl/test/ssl_upgrade_SUITE.erl index 875399db76..ead18aeb73 100644 --- a/lib/ssl/test/ssl_upgrade_SUITE.erl +++ b/lib/ssl/test/ssl_upgrade_SUITE.erl @@ -47,10 +47,7 @@ init_per_suite(Config0) -> {skip, Reason} -> {skip, Reason}; Config -> - Result = - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config), - proplists:get_value(priv_dir, Config)), - ssl_test_lib:cert_options(Config) + ssl_test_lib:make_rsa_cert(Config) end catch _:_ -> {skip, "Crypto did not start"} @@ -149,8 +146,8 @@ use_connection(Socket) -> end. soft_start_connection(Config, ResulProxy) -> - ClientOpts = proplists:get_value(client_verification_opts, Config), - ServerOpts = proplists:get_value(server_verification_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, ResulProxy}, @@ -166,8 +163,8 @@ soft_start_connection(Config, ResulProxy) -> {Server, Client}. restart_start_connection(Config, ResulProxy) -> - ClientOpts = proplists:get_value(client_verification_opts, Config), - ServerOpts = proplists:get_value(server_verification_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = start_server([{node, ServerNode}, {port, 0}, {from, ResulProxy}, diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index d4a4ba268b..6f6849a19d 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -757,7 +757,7 @@ handle_event(_, _, State, Data) -> right before entering the initial state even though this actually is not a <em>state change</em>. In this case <c>OldState =:= State</c>, - which can not happen for a subsequent state change, + which cannot happen for a subsequent state change, but will happen when repeating the <em>state enter call</em>. </p> </desc> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 23c3f6e981..65650a25c7 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.8.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a performance regression when reading files + opened with the <c>compressed</c> flag.</p> + <p> + Own Id: OTP-15706 Aux Id: ERIERL-336 </p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.8</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index c95f7637f7..86003c953d 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -155,8 +155,10 @@ APPUP_TARGET= $(EBIN)/$(APPUP_FILE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native +else +ERL_COMPILE_FLAGS += -Werror endif -ERL_COMPILE_FLAGS += -I../include -I../../kernel/include -Werror +ERL_COMPILE_FLAGS += -I../include -I../../kernel/include # ---------------------------------------------------- # Targets diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 939b1fb488..1504326c61 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -126,11 +126,12 @@ %% per write than base 10, but the speedup is only 21%.) -define(DEFAULT, undefined). --define(LEAFSIZE, 10). % the "base" --define(NODESIZE, ?LEAFSIZE). % (no reason to have a different size) +-define(LEAFSIZE, 10). % the "base" (assumed to be > 1) +-define(NODESIZE, ?LEAFSIZE). % must not be LEAFSIZE-1; keep same as leaf -define(NODEPATTERN(S), {_,_,_,_,_,_,_,_,_,_,S}). % NODESIZE+1 elements! --define(NEW_NODE(S), % beware of argument duplication! - setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(S)),(S))). +-define(NEW_NODE(E,S), % general case (currently unused) + setelement((?NODESIZE+1),erlang:make_tuple((?NODESIZE+1),(E)),(S))). +-define(NEW_NODE(S), erlang:make_tuple((?NODESIZE+1),(S))). % when E = S -define(NEW_LEAF(D), erlang:make_tuple(?LEAFSIZE,(D))). -define(NODELEAFS, ?NODESIZE*?LEAFSIZE). @@ -605,7 +606,7 @@ grow(I, E, M) -> grow_1(I, E, M). grow_1(I, E, M) when I >= M -> - grow(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); + grow_1(I, setelement(1, ?NEW_NODE(M), E), ?extend(M)); grow_1(_I, E, M) -> {E, M}. @@ -1631,12 +1632,11 @@ foldl_test_() -> ?_assert(foldl(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldl(Reverse, [], from_list(lists:seq(0,1000))) =:= lists:reverse(lists:seq(0,1000))), - ?_assert({999,[N0*100+1+2,N0*2+1+1,0]} =:= - foldl(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[N0*100+1+2,N0*2+1+1,0]}, + foldl(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. @@ -1786,12 +1786,11 @@ foldr_test_() -> ?_assert(foldr(Sum, 0, from_list(lists:seq(0,10))) =:= 55), ?_assert(foldr(List, [], from_list(lists:seq(0,1000))) =:= lists:seq(0,1000)), - ?_assert({999,[0,N0*2+1+1,N0*100+1+2]} =:= - foldr(Vals, {0,[]}, + ?_assertEqual({N0*100+1-2,[0,N0*2+1+1,N0*100+1+2]}, + foldr(Vals, {0,[]}, set(N0*100+1,2, set(N0*2+1,1, set(0,0,new()))))) - ]. -endif. diff --git a/lib/stdlib/src/erl_tar.erl b/lib/stdlib/src/erl_tar.erl index d8b8f466b1..7064fcacfa 100644 --- a/lib/stdlib/src/erl_tar.erl +++ b/lib/stdlib/src/erl_tar.erl @@ -324,7 +324,7 @@ do_open(Name, Mode) when is_list(Mode) -> open1({binary,Bin}, read, _Raw, Opts) when is_binary(Bin) -> case file:open(Bin, [ram,binary,read]) of {ok,File} -> - _ = [ram_file:uncompress(File) || Opts =:= [compressed]], + _ = [ram_file:uncompress(File) || lists:member(compressed, Opts)], {ok, #reader{handle=File,access=read,func=fun file_op/2}}; Error -> Error @@ -357,7 +357,7 @@ open_mode([read|Rest], false, Raw, Opts) -> open_mode([write|Rest], false, Raw, Opts) -> open_mode(Rest, write, Raw, Opts); open_mode([compressed|Rest], Access, Raw, Opts) -> - open_mode(Rest, Access, Raw, [compressed|Opts]); + open_mode(Rest, Access, Raw, [compressed,read_ahead|Opts]); open_mode([cooked|Rest], Access, _Raw, Opts) -> open_mode(Rest, Access, [], Opts); open_mode([], Access, Raw, Opts) -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 37ea97c353..08612ed17f 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -41,7 +41,9 @@ {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.8$">>,[restart_new_emulator]}, + {<<"^3\\.8\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -56,4 +58,6 @@ {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^3\\.8$">>,[restart_new_emulator]}, + {<<"^3\\.8\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 2939e78d9d..1f8bdc5432 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1247,18 +1247,20 @@ split_1(Bin, [_C|_]=Needle, Start, Where, Curr0, Acc) -> end end. -lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps, Ts) when is_integer(CP) -> +lexemes_m([CP|_]=Cs0, {GCs,CPs,_}=Seps0, Ts) when is_integer(CP) -> case lists:member(CP, CPs) of true -> [GC|Cs2] = unicode_util:gc(Cs0), case lists:member(GC, GCs) of true -> - lexemes_m(Cs2, Seps, Ts); + lexemes_m(Cs2, Seps0, Ts); false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; false -> + Seps = search_compile(Seps0), {Lexeme,Rest} = lexeme_pick(Cs0, Seps, []), lexemes_m(Rest, Seps, [Lexeme|Ts]) end; diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 8561491d50..87ca9bd32c 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -70,7 +70,10 @@ -export([smp_insert/1, smp_fixed_delete/1, smp_unfix_fix/1, smp_select_delete/1, smp_ordered_iteration/1, smp_select_replace/1, otp_8166/1, otp_8732/1, delete_unfix_race/1]). --export([throughput_benchmark/0, test_throughput_benchmark/1]). +-export([throughput_benchmark/0, + throughput_benchmark/1, + test_throughput_benchmark/1, + long_throughput_benchmark/1]). -export([exit_large_table_owner/1, exit_many_large_table_owner/1, exit_many_tables_owner/1, @@ -93,6 +96,7 @@ -include_lib("stdlib/include/ms_transform.hrl"). % ets:fun2ms -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). -define(m(A,B), assert_eq(A,B)). -define(heap_binary_size, 64). @@ -151,7 +155,8 @@ all() -> take, whereis_table, delete_unfix_race, - test_throughput_benchmark]. + test_throughput_benchmark, + {group, benchmark}]. groups() -> [{new, [], @@ -179,7 +184,9 @@ groups() -> {meta_smp, [], [meta_lookup_unnamed_read, meta_lookup_unnamed_write, meta_lookup_named_read, meta_lookup_named_write, - meta_newdel_unnamed, meta_newdel_named]}]. + meta_newdel_unnamed, meta_newdel_named]}, + {benchmark, [], + [long_throughput_benchmark]}]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -192,9 +199,61 @@ end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false), ok. +init_per_group(benchmark, Config) -> + P = self(), + %% Spawn owner of ETS table that is alive until end_per_group is run + EtsProcess = + spawn( + fun()-> + Tab = ets:new(ets_benchmark_result_summary_tab, [public]), + P ! {the_table, Tab}, + receive + kill -> ok + end + end), + Tab = receive {the_table, T} -> T end, + CounterNames = [nr_of_benchmarks, + total_throughput, + nr_of_set_benchmarks, + total_throughput_set, + nr_of_ordered_set_benchmarks, + total_throughput_ordered_set], + lists:foreach(fun(CtrName) -> + ets:insert(Tab, {CtrName, 0.0}) + end, + CounterNames), + [{ets_benchmark_result_summary_tab, Tab}, + {ets_benchmark_result_summary_tab_process, EtsProcess} | Config]; init_per_group(_GroupName, Config) -> Config. +end_per_group(benchmark, Config) -> + T = proplists:get_value(ets_benchmark_result_summary_tab, Config), + EtsProcess = proplists:get_value(ets_benchmark_result_summary_tab_process, Config), + Report = + fun(NOfBenchmarksCtr, TotThroughoutCtr, Name) -> + Average = + ets:lookup_element(T, TotThroughoutCtr, 2) / + ets:lookup_element(T, NOfBenchmarksCtr, 2), + io:format("~p ~p~n", [Name, Average]), + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value, Average}]}) + end, + Report(nr_of_benchmarks, + total_throughput, + "Average Throughput"), + Report(nr_of_set_benchmarks, + total_throughput_set, + "Average Throughput Set"), + Report(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set, + "Average Throughput Ordered Set"), + ets:delete(T), + EtsProcess ! kill, + Config; end_per_group(_GroupName, Config) -> Config. @@ -6530,8 +6589,8 @@ whereis_table(Config) when is_list(Config) -> ok. -%% The following work functions are used by -%% throughput_benchmark/4. They are declared on the top level beacuse +%% The following help functions are used by +%% throughput_benchmark. They are declared on the top level beacuse %% declaring them as function local funs cause a scalability issue. get_op([{_,O}], _RandNum) -> O; @@ -6566,10 +6625,131 @@ prefill_table_loop(T, RS0, N, ObjFun) -> ets:insert(T, ObjFun(Key)), prefill_table_loop(T, RS1, N-1, ObjFun). -throughput_benchmark() -> - throughput_benchmark(false, not_set, not_set). +-record(ets_throughput_bench_config, + {benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = not_set, + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {1.0, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq10} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq1000} + ], + [ + {1.0, nextseq1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.7999, lookup}, + {0.0001, partial_select1000} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.799999, lookup}, + {0.000001, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public], + [ordered_set, public, {write_concurrency, true}], + [ordered_set, public, {read_concurrency, true}], + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public], + [set, public, {write_concurrency, true}], + [set, public, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun() -> ok end, + verify_etsmem_fun = fun(_) -> true end, + notify_res_fun = fun(_Name, _Throughput) -> ok end, + print_result_paths_fun = + fun(ResultPath, _LatestResultPath) -> + Comment = + io_lib:format("<a href=\"file:///~s\">Result visualization</a>",[ResultPath]), + {comment, Comment} + end + }). + +stdout_notify_res(ResultPath, LatestResultPath) -> + io:format("Result Location: /~s~n", [ResultPath]), + io:format("Latest Result Location: ~s~n", [LatestResultPath]). -throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> +throughput_benchmark() -> + throughput_benchmark( + #ets_throughput_bench_config{ + print_result_paths_fun = fun stdout_notify_res/2}). + +throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = BenchmarkDurationMs, + recover_time_ms = RecoverTimeMs, + thread_counts = ThreadCountsOpt, + key_ranges = KeyRanges, + scenarios = Scenarios, + table_types = TableTypes, + etsmem_fun = ETSMemFun, + verify_etsmem_fun = VerifyETSMemFun, + notify_res_fun = NotifyResFun, + print_result_paths_fun = PrintResultPathsFun}) -> NrOfSchedulers = erlang:system_info(schedulers), %% Definitions of operations that are supported by the benchmark NextSeqOp = @@ -6634,7 +6814,7 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> fun(T,KeyRange) -> NextSeqOp(T,KeyRange,1000) end, selectAll => fun(T,_KeyRange) -> - case -1 =:= ets:select_count(T, ets:fun2ms(fun(X) -> true end)) of + case -1 =:= ets:select_count(T, ets:fun2ms(fun(_X) -> true end)) of true -> io:format("Will never be printed"); false -> ok end @@ -6683,11 +6863,28 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> false -> ok end end, + DataHolder = + fun DataHolderFun(Data)-> + receive + {get_data, Pid} -> Pid ! {ets_bench_data, Data}; + D -> DataHolderFun([Data,D]) + end + end, + DataHolderPid = spawn_link(fun()-> DataHolder([]) end), + PrintData = + fun (Str, List) -> + io:format(Str, List), + DataHolderPid ! io_lib:format(Str, List) + end, + GetData = + fun () -> + DataHolderPid ! {get_data, self()}, + receive {ets_bench_data, Data} -> Data end + end, %% Function that runs a benchmark instance and returns the number %% of operations that were performed RunBenchmark = - fun(NrOfProcs, TableConfig, Scenario, - Range, Duration, RecoverTime) -> + fun({NrOfProcs, TableConfig, Scenario, Range, Duration}) -> ProbHelpTab = CalculateOpsProbHelpTab(Scenario, 0), Table = ets:new(t, TableConfig), Nobj = Range div 2, @@ -6695,16 +6892,15 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> Nobj = ets:info(Table, size), SafeFixTableIfRequired(Table, Scenario, true), ParentPid = self(), + Worker = + fun() -> + receive start -> ok end, + WorksDone = + do_work(0, Table, ProbHelpTab, Range, Operations), + ParentPid ! WorksDone + end, ChildPids = - lists:map( - fun(_N) -> - spawn(fun() -> - receive start -> ok end, - WorksDone = - do_work(0, Table, ProbHelpTab, Range, Operations), - ParentPid ! WorksDone - end) - end, lists:seq(1, NrOfProcs)), + lists:map(fun(_N) ->spawn_link(Worker)end, lists:seq(1, NrOfProcs)), lists:foreach(fun(Pid) -> Pid ! start end, ChildPids), timer:sleep(Duration), lists:foreach(fun(Pid) -> Pid ! stop end, ChildPids), @@ -6716,185 +6912,194 @@ throughput_benchmark(TestMode, BenchmarkRunMs, RecoverTimeMs) -> end, 0, ChildPids), SafeFixTableIfRequired(Table, Scenario, false), ets:delete(Table), - timer:sleep(RecoverTime), TotalWorksDone end, - %% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% Benchmark Configuration %%%%%%%%%%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% - %% Change the following variables to configure the benchmark runs - ThreadCounts = - case TestMode of - true -> [1, NrOfSchedulers]; - false -> CalculateThreadCounts([1]) - end, - KeyRanges = % Sizes of the key ranges - case TestMode of - true -> [50000]; - false -> [1000000] + RunBenchmarkInSepProcess = + fun(ParameterTuple) -> + P = self(), + spawn_link(fun()-> P ! {bench_result, RunBenchmark(ParameterTuple)} end), + Result = receive {bench_result, Res} -> Res end, + timer:sleep(RecoverTimeMs), + Result end, - Duration = - case BenchmarkRunMs of % Duration of a benchmark run in milliseconds - not_set -> 30000; - _ -> BenchmarkRunMs + RunBenchmarkAndReport = + fun(ThreadCount, + TableType, + Scenario, + KeyRange, + Duration) -> + Result = RunBenchmarkInSepProcess({ThreadCount, + TableType, + Scenario, + KeyRange, + Duration}), + Throughput = Result/(Duration/1000.0), + PrintData("; ~f",[Throughput]), + Name = io_lib:format("Scenario: ~w, Key Range Size: ~w, " + "# of Processes: ~w, Table Type: ~w", + [Scenario, KeyRange, ThreadCount, TableType]), + NotifyResFun(Name, Throughput) end, - TimeMsToSleepAfterEachBenchmarkRun = - case RecoverTimeMs of - not_set -> 1000; - _ -> RecoverTimeMs + ThreadCounts = + case ThreadCountsOpt of + not_set -> + CalculateThreadCounts([1]); + _ -> ThreadCountsOpt end, - TableTypes = % The table types that will be benchmarked - [ - [ordered_set, public], - [ordered_set, public, {write_concurrency, true}], - [ordered_set, public, {read_concurrency, true}], - [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], - [set, public], - [set, public, {write_concurrency, true}], - [set, public, {read_concurrency, true}], - [set, public, {write_concurrency, true}, {read_concurrency, true}] - ], - Scenarios = % Benchmark scenarios (the fractions should add up to approximately 1.0) - [ - [ - {0.5, insert}, - {0.5, delete} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.8, lookup} - ], - [ - {0.01, insert}, - {0.01, delete}, - {0.98, lookup} - ], - [ - {1.0, lookup} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq10} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq100} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.4, lookup}, - {0.4, nextseq1000} - ], - [ - {1.0, nextseq1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, selectAll} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.79, lookup}, - {0.01, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.7999, lookup}, - {0.0001, partial_select1000} - ], - [ - {0.1, insert}, - {0.1, delete}, - {0.799999, lookup}, - {0.000001, partial_select1000} - ] - ], - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %%%% End of Benchmark Configuration %%%%%%%%%%%%%%%% - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% Prepare for memory check - EtsMem = case TestMode of - true -> etsmem(); - false -> ok - end, %% Run the benchmark - io:format("# Each instance of the benchmark runs for ~w seconds:~n", [Duration/1000]), - io:format("# The result of a benchmark instance is presented as a number representing~n"), - io:format("# the number of operations performed per second:~n~n~n"), - io:format("# To plot graphs for the results below:~n"), - io:format("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n"), - io:format("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n"), - io:format("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n"), - io:format("# step 1 and press the Render button~n~n"), - io:format("#BENCHMARK STARTED$~n"), + PrintData("# Each instance of the benchmark runs for ~w seconds:~n", [BenchmarkDurationMs/1000]), + PrintData("# The result of a benchmark instance is presented as a number representing~n",[]), + PrintData("# the number of operations performed per second:~n~n~n",[]), + PrintData("# To plot graphs for the results below:~n",[]), + PrintData("# 1. Open \"$ERL_TOP/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html\" in a web browser~n",[]), + PrintData("# 2. Copy the lines between \"#BENCHMARK STARTED$\" and \"#BENCHMARK ENDED$\" below~n",[]), + PrintData("# 3. Paste the lines copied in step 2 to the text box in the browser window opened in~n",[]), + PrintData("# step 1 and press the Render button~n~n",[]), + PrintData("#BENCHMARK STARTED$~n",[]), + EtsMem = ETSMemFun(), %% The following loop runs all benchmark scenarios and prints the results (i.e, operations/second) lists:foreach( fun(KeyRange) -> lists:foreach( fun(Scenario) -> - io:format("Scenario: ~s | Key Range Size: ~w$~n", - [RenderScenario(Scenario, ""), - KeyRange]), + PrintData("Scenario: ~s | Key Range Size: ~w$~n", + [RenderScenario(Scenario, ""), KeyRange]), lists:foreach( fun(ThreadCount) -> - io:format("; ~w",[ThreadCount]) + PrintData("; ~w",[ThreadCount]) end, ThreadCounts), - io:format("$~n",[]), + PrintData("$~n",[]), lists:foreach( fun(TableType) -> - io:format("~w ",[TableType]), + PrintData("~w ",[TableType]), lists:foreach( fun(ThreadCount) -> - Result = RunBenchmark(ThreadCount, + RunBenchmarkAndReport(ThreadCount, TableType, Scenario, KeyRange, - Duration, - TimeMsToSleepAfterEachBenchmarkRun), - io:format("; ~f",[Result/(Duration/1000.0)]) + BenchmarkDurationMs) end, ThreadCounts), - io:format("$~n",[]) + PrintData("$~n",[]) end, TableTypes) end, Scenarios) end, KeyRanges), - io:format("~n#BENCHMARK ENDED$~n~n"), - case TestMode of - true -> verify_etsmem(EtsMem); - false -> ok - end. + PrintData("~n#BENCHMARK ENDED$~n~n",[]), + VerifyETSMemFun(EtsMem), + DataDir = filename:join(filename:dirname(code:which(?MODULE)), "ets_SUITE_data"), + TemplatePath = filename:join(DataDir, "visualize_throughput.html"), + {ok, Template} = file:read_file(TemplatePath), + OutputData = string:replace(Template, "#bench_data_placeholder", GetData()), + OutputPath1 = filename:join(DataDir, "ets_bench_result.html"), + {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_datetime(erlang:timestamp()), + StrTime = lists:flatten(io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w",[Year,Month,Day,Hour,Minute,Second])), + OutputPath2 = filename:join(DataDir, io_lib:format("ets_bench_result_~s.html", [StrTime])), + file:write_file(OutputPath1, OutputData), + file:write_file(OutputPath2, OutputData), + PrintResultPathsFun(OutputPath2, OutputPath1). test_throughput_benchmark(Config) when is_list(Config) -> - throughput_benchmark(true, 100, 0). - + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 100, + recover_time_ms = 0, + thread_counts = [1, erlang:system_info(schedulers)], + key_ranges = [50000], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1}). + +long_throughput_benchmark(Config) when is_list(Config) -> + N = erlang:system_info(schedulers), + throughput_benchmark( + #ets_throughput_bench_config{ + benchmark_duration_ms = 3000, + recover_time_ms = 1000, + thread_counts = [1, N div 2, N], + key_ranges = [1000000], + scenarios = + [ + [ + {0.5, insert}, + {0.5, delete} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.8, lookup} + ], + [ + {0.01, insert}, + {0.01, delete}, + {0.98, lookup} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.4, lookup}, + {0.4, nextseq100} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, selectAll} + ], + [ + {0.1, insert}, + {0.1, delete}, + {0.79, lookup}, + {0.01, partial_select1000} + ] + ], + table_types = + [ + [ordered_set, public, {write_concurrency, true}, {read_concurrency, true}], + [set, public, {write_concurrency, true}, {read_concurrency, true}] + ], + etsmem_fun = fun etsmem/0, + verify_etsmem_fun = fun verify_etsmem/1, + notify_res_fun = + fun(Name, Throughput) -> + SummaryTable = + proplists:get_value(ets_benchmark_result_summary_tab, Config), + AddToSummaryCounter = + case SummaryTable of + undefined -> + fun(_, _) -> + ok + end; + Tab -> + fun(CounterName, ToAdd) -> + OldVal = ets:lookup_element(Tab, CounterName, 2), + NewVal = OldVal + ToAdd, + ets:insert(Tab, {CounterName, NewVal}) + end + end, + Record = + fun(NoOfBenchsCtr, TotThrputCtr) -> + AddToSummaryCounter(NoOfBenchsCtr, 1), + AddToSummaryCounter(TotThrputCtr, Throughput) + end, + Record(nr_of_benchmarks, total_throughput), + case string:find(Name, "ordered_set") of + nomatch -> + Record(nr_of_set_benchmarks, total_throughput_set); + _ -> + Record(nr_of_ordered_set_benchmarks, + total_throughput_ordered_set) + end, + ct_event:notify( + #event{name = benchmark_data, + data = [{suite,"ets_bench"}, + {name, Name}, + {value,Throughput}]}) + end + }). add_lists(L1,L2) -> add_lists(L1,L2,[]). diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html index a2c61aa938..27d6849c60 100644 --- a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html +++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html @@ -42,7 +42,7 @@ </p> Paste the generated data in the field below and press the Render button: <br> - <textarea id="dataField" rows="4" cols="50"></textarea> + <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea> <br> <input type="checkbox" id="barPlot"> Bar Plot <br> @@ -56,13 +56,13 @@ <br> <input type="checkbox" class="showCheck" value="[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[ordered_set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public]"> Show <code>[set,public]</code> + <input type="checkbox" class="showCheck" value="[set,public]" checked> Show <code>[set,public]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]"> Show <code>[set,public,{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{read_concurrency,true}]" checked> Show <code>[set,public,{read_concurrency,true}]</code> <br> - <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]"> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> + <input type="checkbox" class="showCheck" value="[set,public,{write_concurrency,true},{read_concurrency,true}]" checked> Show <code>[set,public,{write_concurrency,true},{read_concurrency,true}]</code> <br> <button id="renderButton" type="button">Render</button> diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec index 4de7c1a0eb..bf64eae2c7 100644 --- a/lib/stdlib/test/stdlib.spec +++ b/lib/stdlib/test/stdlib.spec @@ -2,3 +2,6 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [binary,base64,gen_server,gen_statem,unicode], "Benchmark only"}. +{skip_groups,"../stdlib_test",ets_SUITE, + [benchmark], + "Benchmark only"}. diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec index 7a0da811a0..6d665f22b6 100644 --- a/lib/stdlib/test/stdlib_bench.spec +++ b/lib/stdlib/test/stdlib_bench.spec @@ -8,3 +8,4 @@ {skip_groups,"../stdlib_test",stdlib_bench_SUITE, [gen_server_comparison,gen_statem_comparison], "Not a benchmark"}. +{groups,"../stdlib_test",ets_SUITE,[benchmark]}. diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl index 251e09121c..248912c3f2 100644 --- a/lib/stdlib/test/string_SUITE.erl +++ b/lib/stdlib/test/string_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2018. All Rights Reserved. +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -754,19 +754,22 @@ do_measure(DataDir) -> io:format("~p~n",[byte_size(Bin)]), Do = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, Do2 = fun(Name, Func, Mode) -> {N, Mean, Stddev, _} = time_func(Func, binary, <<>>, 20), - io:format("~15w ~6w ~6.2fms ±~5.2fms #~.2w gc included~n", + io:format("~15w ~15w ~8.2fms ±~6.2fms #~.2w gc included~n", [Name, Mode, Mean/1000, Stddev/1000, N]) end, + %% lefty_list means a list balanced to the left, like + %% [[[30],31],32]. Only some functions check such lists. + Modes = [list, lefty_list, binary, {many_lists,1}, {many_lists, 4}], io:format("----------------------~n"), Do(old_tokens, fun(Str) -> string:tokens(Str, [$\n,$\r]) end, list), Tokens = {lexemes, fun(Str) -> string:lexemes(Str, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Tokens], Mode <- Modes], S0 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....", S0B = <<"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy.....">>, @@ -824,17 +827,17 @@ do_measure(DataDir) -> io:format("--~n",[]), NthTokens = {nth_lexemes, fun(Str) -> string:nth_lexeme(Str, 18000, [$\n,$\r]) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [NthTokens], Mode <- Modes], Do2(take_t, repeat(fun() -> string:take(S0, [$.,$y], false, trailing) end), list), Do2(take_t, repeat(fun() -> string:take(S0B, [$.,$y], false, trailing) end), binary), Do2(take_tc, repeat(fun() -> string:take(S0, [$x], true, trailing) end), list), Do2(take_tc, repeat(fun() -> string:take(S0B, [$x], true, trailing) end), binary), Length = {length, fun(Str) -> string:length(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Length], Mode <- Modes], Reverse = {reverse, fun(Str) -> string:reverse(Str) end}, - [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- [list, binary]], + [Do(Name,Fun,Mode) || {Name,Fun} <- [Reverse], Mode <- Modes], ok. @@ -1064,7 +1067,33 @@ time_func(N,Sum,SumSq, _, _, Res, _) -> {N, Mean, Stdev, Res}. mode(binary, Bin) -> Bin; -mode(list, Bin) -> unicode:characters_to_list(Bin). +mode(list, Bin) -> unicode:characters_to_list(Bin); +mode(lefty_list, Bin) -> + L = unicode:characters_to_list(Bin), + to_left(L); +mode({many_lists, N}, Bin) -> + group(unicode:characters_to_list(Bin), N). + +group([], _N) -> + []; +group(L, N) -> + try lists:split(N, L) of + {L1, L2} -> + [L1 | group(L2, N)] + catch + _:_ -> + [L] + end. + +to_left([]) -> + []; +to_left([H|L]) -> + to_left([H], L). + +to_left(V, []) -> + V; +to_left(V, [H|L]) -> + to_left([V,H], L). %% %% Old string lists Test cases starts here. diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index 70eec1a6f2..8636c69a0d 100755..100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -4,7 +4,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% Copyright Ericsson AB 2017-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -460,17 +460,73 @@ gen_cp(Fd) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), - io:put_chars(Fd, "cp([List|R]) ->\n"), - io:put_chars(Fd, " case cp(List) of\n"), - io:put_chars(Fd, " [] -> cp(R);\n"), - io:put_chars(Fd, " [CP] -> [CP|R];\n"), - io:put_chars(Fd, " [C|R0] -> [C|[R0|R]];\n"), - io:put_chars(Fd, " {error,Error} -> {error,[Error|R]}\n"), - io:put_chars(Fd, " end;\n"), + io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), io:put_chars(Fd, "cp(<<C/utf8, R/binary>>) -> [C|R];\n"), io:put_chars(Fd, "cp(<<>>) -> [];\n"), - io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n\n"), + io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), + io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), + io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<C/utf8, T/binary>>, R) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl(<<>>, R) -> cp(R);\n"), + io:put_chars(Fd, "cpl(<<B/binary>>, R) -> {error,[B|R]}.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont([C|T], R) when is_integer(C) -> [C|cpl_cont2(T, R)];\n"), + io:put_chars(Fd, "cpl_cont([L], R) -> cpl_cont(L, R);\n"), + io:put_chars(Fd, "cpl_cont([L|T], R) -> cpl_cont(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont([], R) -> cpl_1_cont(R);\n"), + io:put_chars(Fd, "cpl_cont(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont2([C|T], R) when is_integer(C) -> [C|cpl_cont3(T, R)];\n"), + io:put_chars(Fd, "cpl_cont2([L], R) -> cpl_cont2(L, R);\n"), + io:put_chars(Fd, "cpl_cont2([L|T], R) -> cpl_cont2(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont2([], R) -> cpl_1_cont2(R);\n"), + io:put_chars(Fd, "cpl_cont2(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_cont3([C], R) when is_integer(C) -> [C|R];\n"), + io:put_chars(Fd, "cpl_cont3([C|T], R) when is_integer(C) -> [C,T|R];\n"), + io:put_chars(Fd, "cpl_cont3([L], R) -> cpl_cont3(L, R);\n"), + io:put_chars(Fd, "cpl_cont3([L|T], R) -> cpl_cont3(L, [T|R]);\n"), + io:put_chars(Fd, "cpl_cont3([], R) -> cpl_1_cont3(R);\n"), + io:put_chars(Fd, "cpl_cont3(T, R) -> [T|R].\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont([C|T]) when is_integer(C) -> [C|cpl_1_cont2(T)];\n"), + io:put_chars(Fd, "cpl_1_cont([L]) -> cpl_1_cont(L);\n"), + io:put_chars(Fd, "cpl_1_cont([L|T]) -> cpl_cont(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont2([C|T]) when is_integer(C) -> [C|cpl_1_cont3(T)];\n"), + io:put_chars(Fd, "cpl_1_cont2([L]) -> cpl_1_cont2(L);\n"), + io:put_chars(Fd, "cpl_1_cont2([L|T]) -> cpl_cont2(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont2(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cpl_1_cont3([C|_]=T) when is_integer(C) -> T;\n"), + io:put_chars(Fd, "cpl_1_cont3([L]) -> cpl_1_cont3(L);\n"), + io:put_chars(Fd, "cpl_1_cont3([L|T]) -> cpl_cont3(L, T);\n"), + io:put_chars(Fd, "cpl_1_cont3(T) -> T.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "%%%\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_bin([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp_no_bin([List]) -> cp_no_bin(List);\n"), + io:put_chars(Fd, "cp_no_bin([List|R]) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_bin([]) -> [];\n"), + io:put_chars(Fd, "cp_no_bin(_) -> binary_found.\n"), + io:put_chars(Fd, "\n"), + io:put_chars(Fd, "cp_no_binl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cp_no_binl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cp_no_binl([List], R) -> cp_no_binl(List, R);\n"), + io:put_chars(Fd, "cp_no_binl([List|T], R) -> cp_no_binl(List, [T|R]);\n"), + io:put_chars(Fd, "cp_no_binl([], R) -> cp_no_bin(R);\n"), + io:put_chars(Fd, "cp_no_binl(_, _) -> binary_found.\n\n"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -481,11 +537,26 @@ gen_gc(Fd, GBP) -> "-spec gc(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, - "gc([CP1, CP2|_]=T)\n" - " when CP1 < 256, CP2 < 256, CP1 =/= $\r -> %% Ascii Fast path\n" - " T;\n" + "gc([]=R) -> R;\n" + "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([$\\r=CP|R0]) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " T -> [CP|T]\n" + " end;\n" + "gc([CP1|T1]=T) when CP1 < 256 ->\n" + " case T1 of\n" + " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " _ -> %% Keep the tail binary.\n" + " case cp_no_bin(T1) of\n" + " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " binary_found -> gc_1(T);\n" + " T4 -> gc_1([CP1|T4])\n" + " end\n" + " end;\n" + "gc(<<>>) -> [];\n" "gc(<<CP1/utf8, Rest/binary>>) ->\n" - " if CP1 < 256, CP1 =/= $\r ->\n" + " if CP1 < 256, CP1 =/= $\\r ->\n" " case Rest of\n" " <<CP2/utf8, _/binary>> when CP2 < 256 -> %% Ascii Fast path\n" " [CP1|Rest];\n" @@ -493,13 +564,12 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" + "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" "gc(Str) ->\n" - " gc_1(cp(Str)).\n\n" - "gc_1([$\\r|R0] = R) ->\n" - " case cp(R0) of % Don't break CRLF\n" - " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" - " _ -> R\n" - " end;\n" + " case cp(Str) of\n" + " {error,_}=Error -> Error;\n" + " CPs -> gc(CPs)\n" + " end.\n" ), GenExtP = fun(Range) -> io:format(Fd, "gc_1~s gc_ext_pict(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -507,7 +577,12 @@ gen_gc(Fd, GBP) -> %% Pick codepoints below 256 (some data knowledge here) {ExtendedPictographicLow,ExtendedPictographicHigh} = lists:splitwith(fun({Start,undefined}) -> Start < 256 end,ExtendedPictographic0), - + io:put_chars(Fd, + "\ngc_1([$\\r|R0] = R) ->\n" + " case cp(R0) of % Don't break CRLF\n" + " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" + " _ -> R\n" + " end;\n"), io:put_chars(Fd, "\n%% Handle control\n"), GenControl = fun(Range) -> io:format(Fd, "gc_1~s R0;\n", [gen_clause(Range)]) end, CRs0 = merge_ranges(maps:get(cr, GBP) ++ maps:get(lf, GBP) ++ maps:get(control, GBP), false), @@ -516,7 +591,14 @@ gen_gc(Fd, GBP) -> %%GenControl(R1),GenControl(R2),GenControl(R3), io:put_chars(Fd, "\n%% Optimize Latin-1\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], - io:format(Fd, "gc_1([CP|R]) when CP < 256 -> gc_extend(R,CP);\n\n", []), + + io:format(Fd, + "gc_1([CP|R]=R0) when CP < 256 ->\n" + " case R of\n" + " [CP2|_] when CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n", + []), io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP @@ -540,7 +622,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), - GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,[CP]);\n", [gen_clause(Range)]) end, + GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, [GenRegional(CP) || CP <- merge_ranges(maps:get(regional_indicator,GBP))], %% io:put_chars(Fd, "%% Handle E_Base\n"), %% GenEBase = fun(Range) -> io:format(Fd, "gc_1~s gc_e_cont(R1,[CP]);\n", [gen_clause(Range)]) end, @@ -552,9 +634,7 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "%% Handle extended_pictographic\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicHigh)], io:put_chars(Fd, "\n%% default clauses\n"), - io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(R, CP);\n"), - io:put_chars(Fd, "gc_1([]) -> [];\n"), - io:put_chars(Fd, "gc_1({error,_}=Error) -> Error.\n\n"), + io:put_chars(Fd, "gc_1([CP|R]) -> gc_extend(cp(R), R, CP).\n\n"), io:put_chars(Fd, "%% Handle Prepend\n"), io:put_chars(Fd, @@ -581,31 +661,24 @@ gen_gc(Fd, GBP) -> "%% To simplify binary handling in libraries the tail should be kept binary\n" "%% and not a lookahead CP\n" ), - io:put_chars(Fd, "gc_extend(T, Acc) ->\n" - " gc_extend(cp(T), T, Acc).\n\n"), io:put_chars(Fd, - "gc_extend([CP|T], T0, Acc0) ->\n" + "gc_extend([CP|T], T0, CP0) ->\n" " case is_extend(CP) of\n" - " false ->\n" - " case Acc0 of\n" - " [Acc] -> [Acc|T0];\n" - " [_|_]=Acc -> [lists:reverse(Acc)|T0];\n" - " Acc -> [Acc|T0]\n" - " end;\n" - " _TrueOrZWJ ->\n" - " case Acc0 of\n" - " [_|_] -> gc_extend(T, [CP|Acc0]);\n" - " Acc -> gc_extend(T, [CP,Acc])\n" - " end\n" + " false -> [CP0|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP,CP0])\n" " end;\n" - "gc_extend([], _, Acc0) ->\n" - " case Acc0 of\n" - " [_]=Acc -> Acc;\n" - " [_|_]=Acc -> [lists:reverse(Acc)];\n" - " Acc -> [Acc]\n" + "gc_extend([], _, CP) -> [CP];\n" + "gc_extend({error,R}, _, CP) -> [CP|R].\n\n"), + io:put_chars(Fd, + "gc_extend2([CP|T], T0, Acc) ->\n" + " case is_extend(CP) of\n" + " false -> [lists:reverse(Acc)|T0]; % losing work done on T\n" + " _TrueOrZWJ -> gc_extend2(cp(T), T, [CP|Acc])\n" " end;\n" - "gc_extend({error,R}, T, Acc0) ->\n" - " gc_extend([], T, Acc0) ++ [R].\n\n" + "gc_extend2([], _, Acc) ->\n" + " [lists:reverse(Acc)];\n" + "gc_extend2({error,R}, _, Acc) ->\n" + " [lists:reverse(Acc)] ++ [R].\n\n" ), [ZWJ] = maps:get(zwj, GBP), GenExtend = fun(R) when R =:= ZWJ -> io:format(Fd, "is_extend~s zwj;\n", [gen_single_clause(ZWJ)]); @@ -660,10 +733,10 @@ gen_gc(Fd, GBP) -> %% -------------------- io:put_chars(Fd, "%% Handle Regional\n"), [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), - io:put_chars(Fd,"gc_regional(R0, Acc) ->\n" + io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend(R1,[CP|Acc]);~n",[RLess, RLarge]), - io:put_chars(Fd," R1 -> gc_extend(R1, R0, Acc)\n" + io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), %% Special hangul @@ -685,16 +758,23 @@ gen_gc(Fd, GBP) -> GenHangulV_2 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulV_2(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul T\n"), io:put_chars(Fd, "gc_h_T(R0, Acc) ->\n case cp(R0) of\n"), GenHangulT_1 = fun(Range) -> io:format(Fd, "~8c~s gc_h_T(R1,[CP|Acc]);\n", [$\s,gen_case_clause(Range)]) end, [GenHangulT_1(CP) || CP <- merge_ranges(maps:get(t,GBP))], - io:put_chars(Fd, " R1 -> gc_extend(R1, R0, Acc)\n end.\n\n"), - - io:put_chars(Fd, "gc_h_lv_lvt({error,_}=Error, Acc) -> gc_extend(Error, [], Acc);\n"), + io:put_chars(Fd, + " R1 ->\n" + " case Acc of\n" + " [CP] -> gc_extend(R1, R0, CP);\n" + " _ -> gc_extend2(R1, R0, Acc)\n" + " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, @@ -703,8 +783,10 @@ gen_gc(Fd, GBP) -> GenHangulLVT = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_T(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLVT(CP) || CP <- merge_ranges(maps:get(lvt,GBP))], - io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(R, CP);\n"), %% From gc_1/1 - io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend(R, Acc).\n\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|R], []) -> gc_extend(cp(R), R, CP);\n"), %% From gc_1/1 + io:put_chars(Fd, "%% Also handles error tuples\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, [CP]) -> gc_extend(R, R, CP);\n"), + io:put_chars(Fd, "gc_h_lv_lvt(R, Acc) -> gc_extend2(R, R, Acc).\n\n"), ok. gen_compose_pairs(Fd, ExclData, Data) -> diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index cbefd6590a..6471dc70e0 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.8 +STDLIB_VSN = 3.8.1 diff --git a/lib/syntax_tools/test/syntax_tools_SUITE.erl b/lib/syntax_tools/test/syntax_tools_SUITE.erl index 6b42f7a0a1..e1dd1bd73b 100644 --- a/lib/syntax_tools/test/syntax_tools_SUITE.erl +++ b/lib/syntax_tools/test/syntax_tools_SUITE.erl @@ -74,7 +74,7 @@ smoke_test_file(File) -> [print_error_markers(F, File) || F <- Forms], ok; {error,Reason} -> - io:format("~s: ~p\n", [File,Reason]), + io:format("~ts: ~p\n", [File,Reason]), error end. @@ -82,7 +82,7 @@ print_error_markers(F, File) -> case erl_syntax:type(F) of error_marker -> {L,M,Info} = erl_syntax:error_marker_info(F), - io:format("~ts:~p: ~s", [File,L,M:format_error(Info)]); + io:format("~ts:~p: ~ts", [File,L,M:format_error(Info)]); _ -> ok end. @@ -362,7 +362,7 @@ test_comment_scan([File|Files],DataDir) -> end, Fs1 = erl_recomment:recomment_forms(Fs0, Comments), Fs2 = erl_syntax_lib:map(Fun, Fs1), - io:format("File: ~s~n", [Filename]), + io:format("File: ~ts~n", [Filename]), io:put_chars(erl_prettypr:format(Fs2, [{paper, 120}, {ribbon, 110}])), test_comment_scan(Files,DataDir). @@ -377,8 +377,8 @@ test_prettypr([File|Files],DataDir,PrivDir) -> PP = erl_prettypr:format(Fs, [{paper, 120}, {ribbon, 110}]), io:put_chars(PP), OutFile = filename:join(PrivDir, File), - ok = file:write_file(OutFile,iolist_to_binary(PP)), - io:format("Parsing OutFile: ~s~n", [OutFile]), + ok = file:write_file(OutFile,unicode:characters_to_binary(PP)), + io:format("Parsing OutFile: ~ts~n", [OutFile]), {ok, Fs2} = epp:parse_file(OutFile, [], []), case [Error || {error, _} = Error <- Fs2] of [] -> @@ -445,7 +445,7 @@ pretty_print_parse_forms([{Fs0,Type}|FsForms],PrivDir,Filename) -> {Fs2,{CC,CT}} = erl_syntax_lib:mapfold(Comment,{0,0}, Fs1), io:format("Commented on ~w cases and ~w tries~n", [CC,CT]), PP = erl_prettypr:format(Fs2), - ok = file:write_file(OutFile,iolist_to_binary(PP)), + ok = file:write_file(OutFile,unicode:characters_to_binary(PP)), pretty_print_parse_forms(FsForms,PrivDir,Filename). diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 3cbe9daa60..38c0eba92b 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -905,8 +905,10 @@ resulting regexp is surrounded by \\_< and \\_>." "dist_get_stat" "dist_ctrl_get_data" "dist_ctrl_get_data_notification" + "dist_ctrl_get_opt" "dist_ctrl_input_handler" "dist_ctrl_put_data" + "dist_ctrl_set_opt" "dmonitor_node" "dt_append_vm_tag_data" "dt_get_tag" diff --git a/lib/tools/emacs/erldoc.el b/lib/tools/emacs/erldoc.el index 770ab299ee..bc16d7a14d 100644 --- a/lib/tools/emacs/erldoc.el +++ b/lib/tools/emacs/erldoc.el @@ -89,6 +89,13 @@ up the indexing." :type 'file :group 'erldoc) +(defcustom erldoc-no-signature-function #'ignore + "Notification function called if no function signature was found." + :type '(choice (function-item :tag "Ignore" ignore) + (function-item :tag "Warn" warn) + (function-item :tag "Error" error)) + :group 'erldoc) + (defun erldoc-strip-string (s) (let* ((re "[ \t\n\r\f\v\u00a0]+") (from (if (string-match (concat "\\`" re) s) (match-end 0) 0)) @@ -212,11 +219,21 @@ up the indexing." ;; Get the full function signature. (when (and (eq (car-safe d) 'a) (gethash (erldoc-dom-get-attribute d 'name) table)) - (push (append (gethash (erldoc-dom-get-attribute d 'name) table) - (list (funcall span-content - (or (erldoc-dom-get-element d 'span) - (cadr (memq d erldoc-dom-walk-siblings)))))) - entries)) + (let* ((name (erldoc-dom-get-attribute d 'name)) + (mfa-url (gethash name table)) + (mfa (car mfa-url)) + (sig (or (funcall span-content d) + (funcall span-content + (or (erldoc-dom-get-element d 'span) + (cadr + (memq d erldoc-dom-walk-siblings)))) + (progn + (funcall erldoc-no-signature-function + "erldoc-parse-man: no sig for %s" + mfa) + nil)))) + (push (append mfa-url (list sig)) + entries))) ;; Get data types (when (and (eq (car-safe d) 'a) (string-prefix-p "type-" @@ -280,7 +297,7 @@ up the indexing." (unless (file-exists-p of) (erldoc-parse-all erldoc-man-index of)) (unless (string= erldoc-output-file of) - (make-symbolic-link of erldoc-output-file)))) + (make-symbolic-link (expand-file-name of) erldoc-output-file)))) (setq erldoc-lookup-table (with-temp-buffer (insert-file-contents erldoc-output-file) @@ -356,9 +373,12 @@ up the indexing." (sigs)) (maphash (lambda (k v) (when (string-match re k) - (push (cons (string-to-number (match-string 1 k)) - (cdr (erldoc-tokenize-signature (cadr v)))) - sigs))) + (if (cadr v) + (push (cons (string-to-number (match-string 1 k)) + (cdr (erldoc-tokenize-signature (cadr v)))) + sigs) + (funcall erldoc-no-signature-function + "erldoc-format-signature: No sig for %s" k)))) (erldoc-lookup-table)) (when sigs ;; Mostly single return type but there are exceptions such as diff --git a/lib/wx/src/Makefile b/lib/wx/src/Makefile index 21b45af2c4..52f4008e0a 100644 --- a/lib/wx/src/Makefile +++ b/lib/wx/src/Makefile @@ -111,10 +111,10 @@ $(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk Makefile # Rules $(EBIN)/%.beam: $(ESRC)/%.erl $(HEADER_FILES) - $(V_ERLC) -W -bbeam $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + $(V_ERLC) -W -bbeam $(ERL_COMPILE_FLAGS) -o$(EBIN) $< $(EBIN)/%.beam: $(EGEN)/%.erl $(HEADER_FILES) - $(V_ERLC) -W -bbeam $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + $(V_ERLC) -W -bbeam $(ERL_COMPILE_FLAGS) -o$(EBIN) $< # ---------------------------------------------------- # Release Target diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index 7f6874e36b..d6b6dfdfb5 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -32,6 +32,28 @@ <p>This document describes the changes made to the Xmerl application.</p> +<section><title>Xmerl 1.3.20</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Handling of character references in attributes are fixed.</p> + <p> + Own Id: OTP-15684 Aux Id: ERL-837 </p> + </item> + <item> + <p> + Normalization of whitespace characters in attributes are + fixed so it works when character references are used.</p> + <p> + Own Id: OTP-15685 Aux Id: ERL-475 </p> + </item> + </list> + </section> + +</section> + <section><title>Xmerl 1.3.19</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/xmerl/src/xmerl_scan.erl b/lib/xmerl/src/xmerl_scan.erl index e543a5a11e..d76ed5c820 100644 --- a/lib/xmerl/src/xmerl_scan.erl +++ b/lib/xmerl/src/xmerl_scan.erl @@ -2410,15 +2410,22 @@ scan_att_chars("&" ++ T, S0, Delim, Acc, TmpAcc,AT,IsNorm) -> % Reference true -> scan_att_chars(T1,S1,Delim,[ExpRef|Acc],[ExpRef|TmpAcc],AT,IsNorm); _ -> - Ch = string_to_char_set(S#xmerl_scanner.encoding, ExpRef), case T of "#" ++ _ -> %% normalization rules (sec 3.3.3) require that for %% character references, the referenced character be %% added directly to the normalized value - scan_att_chars(T1, S1, Delim, Ch ++ Acc,TmpAcc, AT,IsNorm); + {T2,S2,IsNorm2} = + if + ?whitespace(hd(ExpRef)) -> + normalize(T1, S1, IsNorm); + true -> + {T1, S1, IsNorm} + end, + scan_att_chars(T2, S2, Delim, ExpRef ++ Acc, TmpAcc, AT, IsNorm2); _ -> - scan_att_chars(Ch ++ T1, S1, Delim, Acc,TmpAcc, AT,IsNorm) + Ch = string_to_char_set(S#xmerl_scanner.encoding, ExpRef), + scan_att_chars(Ch ++ T1, S1, Delim, Acc, TmpAcc, AT, IsNorm) end end; scan_att_chars("<" ++ _T, S0, _Delim, _Acc,_, _,_) -> % Tags not allowed here @@ -3964,7 +3971,7 @@ normalize(T,S,IsNorm) -> {_,T,S} -> {T,S,IsNorm}; {_,T1,S1} -> - {T1,S1,true} + normalize(T1,S1,true) end. diff --git a/lib/xmerl/vsn.mk b/lib/xmerl/vsn.mk index b6486681c2..31ffa6e749 100644 --- a/lib/xmerl/vsn.mk +++ b/lib/xmerl/vsn.mk @@ -1 +1 @@ -XMERL_VSN = 1.3.19 +XMERL_VSN = 1.3.20 |