diff options
Diffstat (limited to 'lib')
246 files changed, 12395 insertions, 4802 deletions
diff --git a/lib/asn1/doc/src/notes.xml b/lib/asn1/doc/src/notes.xml index 26640acabc..5399528271 100644 --- a/lib/asn1/doc/src/notes.xml +++ b/lib/asn1/doc/src/notes.xml @@ -32,6 +32,40 @@ <p>This document describes the changes made to the asn1 application.</p> +<section><title>Asn1 5.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Default values now work in extension for PER, so if you + give the atom <c>asn1_DEFAULT</c> instead of a value it + will become the default value.</p> + <p> + Own Id: OTP-13011 Aux Id: ERIERL-60 </p> + </item> + </list> + </section> + +</section> + +<section><title>Asn1 5.0.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed compilation error of generated code caused by a + missing quotation of function names as part of an + external call for encoding.</p> + <p> + Own Id: OTP-14519 Aux Id: ERIERL-49 </p> + </item> + </list> + </section> + +</section> + <section><title>Asn1 5.0</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/asn1/src/asn1ct_constructed_per.erl b/lib/asn1/src/asn1ct_constructed_per.erl index 3f1be4febb..aff383479b 100644 --- a/lib/asn1/src/asn1ct_constructed_per.erl +++ b/lib/asn1/src/asn1ct_constructed_per.erl @@ -985,9 +985,11 @@ gen_enc_components_call1(Gen, TopType, [C|Rest], DynamicEnc, Ext) -> Imm1; 'OPTIONAL' -> enc_absent(Gen, Element, [asn1_NOVALUE], Imm1); - {'DEFAULT',Def} -> + {'DEFAULT',Def} when Ext =:= noext -> DefValues = def_values(Type, Def), - enc_absent(Gen, Element, DefValues, Imm1) + enc_absent(Gen, Element, DefValues, Imm1); + {'DEFAULT',_} -> + enc_absent(Gen, Element, [asn1_DEFAULT], Imm1) end, Imm = case Imm2 of [] -> []; diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl index 838d59a512..806f8420ec 100644 --- a/lib/asn1/src/asn1ct_gen.erl +++ b/lib/asn1/src/asn1ct_gen.erl @@ -145,27 +145,37 @@ pgen_n2nconversion(_Erules,#typedef{name=TypeName,typespec=#type{def={'ENUMERATE pgen_n2nconversion(_Erules,_) -> true. -pgen_name2numfunc(_TypeName,[], _) -> +pgen_name2numfunc(TypeNameAsAtom,Mapping,Ext) when is_atom(TypeNameAsAtom) -> + FuncName = list_to_atom("name2num_"++atom_to_list(TypeNameAsAtom)), + pgen_name2numfunc1(FuncName,Mapping,Ext). + +pgen_name2numfunc1(_FuncName,[], _) -> true; -pgen_name2numfunc(TypeName,[{Atom,Number}], extension_marker) -> - emit(["name2num_",TypeName,"(",{asis,Atom},") ->",Number,";",nl]), - emit(["name2num_",TypeName,"({asn1_enum, Num}) -> Num.",nl,nl]); -pgen_name2numfunc(TypeName,[{Atom,Number}], _) -> - emit(["name2num_",TypeName,"(",{asis,Atom},") ->",Number,".",nl,nl]); -pgen_name2numfunc(TypeName,[{Atom,Number}|NNRest], EM) -> - emit(["name2num_",TypeName,"(",{asis,Atom},") ->",Number,";",nl]), - pgen_name2numfunc(TypeName,NNRest, EM). - -pgen_num2namefunc(_TypeName,[], _) -> +pgen_name2numfunc1(FuncName,[{Atom,Number}], extension_marker) -> + emit([{asis,FuncName},"(",{asis,Atom},") ->",Number,";",nl]), + emit([{asis,FuncName},"({asn1_enum, Num}) -> Num.",nl,nl]); +pgen_name2numfunc1(FuncName,[{Atom,Number}], _) -> + emit([{asis,FuncName},"(",{asis,Atom},") ->",Number,".",nl,nl]); +pgen_name2numfunc1(FuncName,[{Atom,Number}|NNRest], EM) -> + emit([{asis,FuncName},"(",{asis,Atom},") ->",Number,";",nl]), + pgen_name2numfunc1(FuncName,NNRest, EM). + +pgen_num2namefunc(TypeNameAsAtom,Mapping,Ext) when is_atom(TypeNameAsAtom) -> + FuncName = list_to_atom("num2name_"++atom_to_list(TypeNameAsAtom)), + pgen_num2namefunc1(FuncName,Mapping,Ext). + +pgen_num2namefunc1(_FuncName,[], _) -> true; -pgen_num2namefunc(TypeName,[{Atom,Number}], extension_marker) -> - emit(["num2name_",TypeName,"(",Number,") ->",{asis,Atom},";",nl]), - emit(["num2name_",TypeName,"(ExtensionNum) -> {asn1_enum, ExtensionNum}.",nl,nl]); -pgen_num2namefunc(TypeName,[{Atom,Number}], _) -> - emit(["num2name_",TypeName,"(",Number,") ->",{asis,Atom},".",nl,nl]); -pgen_num2namefunc(TypeName,[{Atom,Number}|NNRest], EM) -> - emit(["num2name_",TypeName,"(",Number,") ->",{asis,Atom},";",nl]), - pgen_num2namefunc(TypeName,NNRest, EM). +pgen_num2namefunc1(FuncName,[{Atom,Number}], extension_marker) -> + emit([{asis,FuncName},"(",Number,") ->",{asis,Atom},";",nl]), + emit([{asis,FuncName},"(ExtensionNum) -> {asn1_enum, ExtensionNum}.",nl,nl]); +pgen_num2namefunc1(FuncName,[{Atom,Number}], _) -> + emit([{asis,FuncName},"(",Number,") ->",{asis,Atom},".",nl,nl]); +pgen_num2namefunc1(FuncName,[{Atom,Number}|NNRest], EM) -> + emit([{asis,FuncName},"(",Number,") ->",{asis,Atom},";",nl]), + pgen_num2namefunc1(FuncName,NNRest, EM). + + pgen_objects(_,_,_,[]) -> true; diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl index 28b4e46b0c..82e9326294 100644 --- a/lib/asn1/src/asn1ct_gen_per.erl +++ b/lib/asn1/src/asn1ct_gen_per.erl @@ -101,7 +101,7 @@ gen_encode_user(Erules,D) when is_record(D,typedef) -> #'Externaltypereference'{module=CurrMod,type=Etype} -> emit([{asis,enc_func(Etype)},"(Val).",nl]); #'Externaltypereference'{module=Emod,type=Etype} -> - emit([{asis,Emod},":",enc_func(Etype),"(Val).",nl]) + emit([{asis,Emod},":",{asis,enc_func(Etype)},"(Val).",nl]) end. diff --git a/lib/asn1/src/asn1rtt_per_common.erl b/lib/asn1/src/asn1rtt_per_common.erl index 2ecc9e4bc7..5b5f47dfee 100644 --- a/lib/asn1/src/asn1rtt_per_common.erl +++ b/lib/asn1/src/asn1rtt_per_common.erl @@ -542,6 +542,7 @@ extension_bitmap(_Val, Pos, Limit, Acc) when Pos >= Limit -> extension_bitmap(Val, Pos, Limit, Acc) -> Bit = case element(Pos, Val) of asn1_NOVALUE -> 0; + asn1_DEFAULT -> 0; _ -> 1 end, extension_bitmap(Val, Pos+1, Limit, (Acc bsl 1) bor Bit). diff --git a/lib/asn1/test/Makefile b/lib/asn1/test/Makefile index f4041fa89b..c38d1c6ebd 100644 --- a/lib/asn1/test/Makefile +++ b/lib/asn1/test/Makefile @@ -43,6 +43,7 @@ MODULES= \ testChoTypeRefSet \ testConstraints \ testDef \ + testExtensionDefault \ testOpt \ testSeqDefault \ testSeqExtension \ diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index 5fe6945ff2..c61cecca4c 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -147,6 +147,7 @@ groups() -> testImport, testDER, testDEFAULT, + testExtensionDefault, testMvrasn6, testContextSwitchingTypes, testOpenTypeImplicitTag, @@ -444,6 +445,12 @@ testDEFAULT(Config, Rule, Opts) -> testDef:main(Rule), testSeqSetDefaultVal:main(Rule, Opts). +testExtensionDefault(Config) -> + test(Config, fun testExtensionDefault/3). +testExtensionDefault(Config, Rule, Opts) -> + asn1_test_lib:compile_all(["ExtensionDefault"], Config, [Rule|Opts]), + testExtensionDefault:main(Rule). + testMaps(Config) -> test(Config, fun testMaps/3, [{ber,[maps,no_ok_wrapper]}, @@ -1198,14 +1205,14 @@ testComment(Config) -> testName2Number(Config) -> N2NOptions0 = [{n2n,Type} || - Type <- ['CauseMisc', 'CauseProtocol', - 'CauseRadioNetwork', - 'CauseTransport','CauseNas']], + Type <- ['Cause-Misc', 'CauseProtocol']], N2NOptions = [?NO_MAPS_MODULE|N2NOptions0], - asn1_test_lib:compile("S1AP-IEs", Config, N2NOptions), + asn1_test_lib:compile("EnumN2N", Config, N2NOptions), - 0 = 'S1AP-IEs':name2num_CauseMisc('control-processing-overload'), - 'unknown-PLMN' = 'S1AP-IEs':num2name_CauseMisc(5), + 0 = 'EnumN2N':'name2num_Cause-Misc'('control-processing-overload'), + 'unknown-PLMN' = 'EnumN2N':'num2name_Cause-Misc'(5), + 4 = 'EnumN2N':name2num_CauseProtocol('semantic-error'), + 'transfer-syntax-error' = 'EnumN2N':num2name_CauseProtocol(0), %% OTP-10144 %% Test that n2n option generates name2num and num2name functions supporting diff --git a/lib/asn1/test/asn1_SUITE_data/EnumN2N.asn1 b/lib/asn1/test/asn1_SUITE_data/EnumN2N.asn1 index a724f2f3f5..a610eb6230 100644 --- a/lib/asn1/test/asn1_SUITE_data/EnumN2N.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/EnumN2N.asn1 @@ -1,6 +1,28 @@ EnumN2N DEFINITIONS AUTOMATIC TAGS ::= BEGIN +Cause-Misc ::= ENUMERATED { + control-processing-overload, + not-enough-user-plane-processing-resources, + hardware-failure, + om-intervention, + unspecified, + unknown-PLMN, +... +} + +CauseProtocol ::= ENUMERATED { + transfer-syntax-error, + abstract-syntax-error-reject, + abstract-syntax-error-ignore-and-notify, + message-not-compatible-with-receiver-state, + semantic-error, + abstract-syntax-error-falsely-constructed-message, + unspecified, + ... +} + + NoExt ::= ENUMERATED { blue(0), red(1), diff --git a/lib/asn1/test/asn1_SUITE_data/ExtensionDefault.asn1 b/lib/asn1/test/asn1_SUITE_data/ExtensionDefault.asn1 new file mode 100644 index 0000000000..67d9cb6312 --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/ExtensionDefault.asn1 @@ -0,0 +1,12 @@ +ExtensionDefault DEFINITIONS AUTOMATIC TAGS ::= + +BEGIN + +Message ::= SEQUENCE { + id INTEGER (0..5), + ..., + priority Priority DEFAULT low +} +Priority ::= ENUMERATED { low(0), high(1), ... } + +END diff --git a/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 b/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 index 32b8f75dde..dee3cd5048 100644 --- a/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ImportsFrom.asn1 @@ -1,8 +1,9 @@ ImportsFrom DEFINITIONS AUTOMATIC TAGS ::= BEGIN -IMPORTS Int FROM ImportsFrom2; +IMPORTS Int, Quoted-Seq FROM ImportsFrom2; i Int ::= 42 +My-Seq ::= Quoted-Seq END diff --git a/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 b/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 index b0c29d24ae..a8e619e215 100644 --- a/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ImportsFrom2.asn1 @@ -2,6 +2,11 @@ ImportsFrom2 DEFINITIONS AUTOMATIC TAGS ::= BEGIN IMPORTS Int FROM ImportsFrom3; +Quoted-Seq ::= SEQUENCE { + x INTEGER(0..17), + y INTEGER(0..666) +} + LocalDef ::= OCTET STRING END diff --git a/lib/asn1/test/testExtensionDefault.erl b/lib/asn1/test/testExtensionDefault.erl new file mode 100644 index 0000000000..cc50fa95b8 --- /dev/null +++ b/lib/asn1/test/testExtensionDefault.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(testExtensionDefault). + +-export([main/1]). + +main(_Erule) -> + roundtrip('Message', {'Message',1,low}), %Will be explicitly encoded. + roundtrip('Message', {'Message',1,high}), + roundtrip('Message', {'Message',1,asn1_DEFAULT}, {'Message',1,low}), + + map_roundtrip('Message', #{id=>1,priority=>low}), %Will be explicitly encoded. + map_roundtrip('Message', #{id=>1,priority=>high}), + map_roundtrip('Message', #{id=>1}, #{id=>1,priority=>low}), + ok. + +roundtrip(Type, Value) -> + asn1_test_lib:roundtrip('ExtensionDefault', Type, Value). + +roundtrip(Type, Value, Expected) -> + %% asn1_test_lib:roundtrip/3 will invoke map_roundtrip/3, which will + %% not work in this case. Therefore, implement the roundtrip ourselves. + M = 'ExtensionDefault', + {ok,Enc} = M:encode(Type, Value), + {ok,Expected} = M:decode(Type, Enc), + ok. + +map_roundtrip(Type, Value) -> + map_roundtrip(Type, Value, Value). + +map_roundtrip(Type, Value, Expected) -> + M = 'maps_ExtensionDefault', + Enc = M:encode(Type, Value), + Expected = M:decode(Type, Enc), + ok. diff --git a/lib/asn1/vsn.mk b/lib/asn1/vsn.mk index 7329a9f879..5900f3037e 100644 --- a/lib/asn1/vsn.mk +++ b/lib/asn1/vsn.mk @@ -1 +1 @@ -ASN1_VSN = 5.0 +ASN1_VSN = 5.0.2 diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 28b2d44168..37a1846160 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,26 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.15.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + In OTP-20.0, the behavior of c, make, and ct_make was + changed so that in some cases the beam files by default + would be written to the directory where the source files + were found. This is now changed back to the old behavior + so beam files are by default written to current + directory.</p> + <p> + Own Id: OTP-14489 Aux Id: ERL-438 </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.15</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/src/ct_make.erl b/lib/common_test/src/ct_make.erl index 4d66796b83..220cb0473d 100644 --- a/lib/common_test/src/ct_make.erl +++ b/lib/common_test/src/ct_make.erl @@ -280,15 +280,47 @@ recompile(File, NoExec, Load, Opts) -> do_recompile(_File, true, _Load, _Opts) -> out_of_date; -do_recompile(File, false, noload, Opts) -> +do_recompile(File, false, Load, Opts) -> io:format("Recompile: ~ts\n",[File]), - compile:file(File, [report_errors, report_warnings, error_summary |Opts]); -do_recompile(File, false, load, Opts) -> - io:format("Recompile: ~ts\n",[File]), - c:c(File, Opts); -do_recompile(File, false, netload, Opts) -> - io:format("Recompile: ~ts\n",[File]), - c:nc(File, Opts). + case compile:file(File, [report_errors, report_warnings |Opts]) of + Ok when is_tuple(Ok), element(1,Ok)==ok -> + maybe_load(element(2,Ok), Load, Opts); + _Error -> + error + end. + +maybe_load(_Mod, noload, _Opts) -> + ok; +maybe_load(Mod, Load, Opts) -> + %% We have compiled File with options Opts. Find out where the + %% output file went to, and load it. + case compile:output_generated(Opts) of + true -> + Dir = proplists:get_value(outdir,Opts,"."), + do_load(Dir, Mod, Load); + false -> + io:format("** Warning: No object file created - nothing loaded **~n"), + ok + end. + +do_load(Dir, Mod, load) -> + code:purge(Mod), + case code:load_abs(filename:join(Dir, Mod),Mod) of + {module,Mod} -> + {ok,Mod}; + Other -> + Other + end; +do_load(Dir, Mod, netload) -> + Obj = atom_to_list(Mod) ++ code:objfile_extension(), + Fname = filename:join(Dir, Obj), + case file:read_file(Fname) of + {ok,Bin} -> + rpc:eval_everywhere(code,load_binary,[Mod,Fname,Bin]), + {ok,Mod}; + Other -> + Other + end. exists(File) -> case file:read_file_info(File) of diff --git a/lib/common_test/test_server/ts_install.erl b/lib/common_test/test_server/ts_install.erl index c4e0223ac7..c5631fb9c3 100644 --- a/lib/common_test/test_server/ts_install.erl +++ b/lib/common_test/test_server/ts_install.erl @@ -408,17 +408,13 @@ off_heap_msgq() -> end. schedulers() -> - case catch erlang:system_info(smp_support) of - true -> - case {erlang:system_info(schedulers), - erlang:system_info(schedulers_online)} of - {S,S} -> - "/S"++integer_to_list(S); - {S,O} -> - "/S"++integer_to_list(S) ++ ":" ++ - integer_to_list(O) - end; - _ -> "" + case {erlang:system_info(schedulers), + erlang:system_info(schedulers_online)} of + {S,S} -> + "/S"++integer_to_list(S); + {S,O} -> + "/S"++integer_to_list(S) ++ ":" ++ + integer_to_list(O) end. bind_type() -> diff --git a/lib/common_test/test_server/ts_run.erl b/lib/common_test/test_server/ts_run.erl index 82ae44ec06..ec4a54b249 100644 --- a/lib/common_test/test_server/ts_run.erl +++ b/lib/common_test/test_server/ts_run.erl @@ -204,11 +204,7 @@ make_command(Vars, Spec, State) -> _ -> ok end, - "cerl -valgrind" ++ - case erlang:system_info(smp_support) of - true -> " -smp"; - false -> "" - end + "cerl -valgrind" end, Naming = case ts_lib:var(longnames, Vars) of diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index a219aa4736..9fc3f7f797 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.15 +COMMON_TEST_VSN = 1.15.1 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index f3d42a909b..bc335a9eaa 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,23 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fail labels on guard BIFs weren't taken into account + during an optimization pass, and a bug in the validation + pass sometimes prevented this from being noticed when a + fault occurred.</p> + <p> + Own Id: OTP-14522 Aux Id: ERIERL-48 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index ef6db66ff6..9b22e5197b 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -83,6 +83,7 @@ MODULES = \ core_scan \ erl_bifs \ rec_env \ + sys_core_alias \ sys_core_bsm \ sys_core_dsetel \ sys_core_fold \ @@ -194,6 +195,7 @@ $(EBIN)/core_lib.beam: core_parse.hrl $(EBIN)/core_lint.beam: core_parse.hrl $(EBIN)/core_parse.beam: core_parse.hrl $(EGEN)/core_parse.erl $(EBIN)/core_pp.beam: core_parse.hrl +$(EBIN)/sys_core_alias.beam: core_parse.hrl $(EBIN)/sys_core_dsetel.beam: core_parse.hrl $(EBIN)/sys_core_fold.beam: core_parse.hrl $(EBIN)/sys_core_fold_lists.beam: core_parse.hrl diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index b736d39f9c..e094c2c320 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -24,7 +24,7 @@ -export([module/2]). -export([bs_clean_saves/1]). -export([clean_labels/1]). --import(lists, [map/2,foldl/3,reverse/1,filter/2]). +-import(lists, [foldl/3,reverse/1,filter/2]). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -118,7 +118,7 @@ add_to_work_list(F, {Fs,Used}=Sets) -> clean_labels(Fs0) -> St0 = #st{lmap=[],entry=1,lc=1}, {Fs1,#st{lmap=Lmap0,lc=Lc}} = function_renumber(Fs0, St0, []), - Lmap = gb_trees:from_orddict(ordsets:from_list(Lmap0)), + Lmap = maps:from_list(Lmap0), Fs = function_replace(Fs1, Lmap, []), {Fs,Lc}. @@ -187,7 +187,8 @@ is_record_tuple(_, _, _) -> no. function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) -> Asm = try - replace(Asm0, [], Dict) + Fb = fun(Old) -> throw({error,{undefined_label,Old}}) end, + beam_utils:replace_labels(Asm0, [], Dict, Fb) catch throw:{error,{undefined_label,Lbl}=Reason} -> io:format("Function ~s/~w refers to undefined label ~w\n", @@ -197,57 +198,6 @@ function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) -> function_replace(Fs, Dict, [{function,Name,Arity,Entry,Asm}|Acc]); function_replace([], _, Acc) -> Acc. -replace([{test,Test,{f,Lbl},Ops}|Is], Acc, D) -> - replace(Is, [{test,Test,{f,label(Lbl, D)},Ops}|Acc], D); -replace([{test,Test,{f,Lbl},Live,Ops,Dst}|Is], Acc, D) -> - replace(Is, [{test,Test,{f,label(Lbl, D)},Live,Ops,Dst}|Acc], D); -replace([{select,I,R,{f,Fail0},Vls0}|Is], Acc, D) -> - Vls = map(fun ({f,L}) -> {f,label(L, D)}; - (Other) -> Other - end, Vls0), - Fail = label(Fail0, D), - replace(Is, [{select,I,R,{f,Fail},Vls}|Acc], D); -replace([{'try',R,{f,Lbl}}|Is], Acc, D) -> - replace(Is, [{'try',R,{f,label(Lbl, D)}}|Acc], D); -replace([{'catch',R,{f,Lbl}}|Is], Acc, D) -> - replace(Is, [{'catch',R,{f,label(Lbl, D)}}|Acc], D); -replace([{jump,{f,Lbl}}|Is], Acc, D) -> - replace(Is, [{jump,{f,label(Lbl, D)}}|Acc], D); -replace([{loop_rec,{f,Lbl},R}|Is], Acc, D) -> - replace(Is, [{loop_rec,{f,label(Lbl, D)},R}|Acc], D); -replace([{loop_rec_end,{f,Lbl}}|Is], Acc, D) -> - replace(Is, [{loop_rec_end,{f,label(Lbl, D)}}|Acc], D); -replace([{wait,{f,Lbl}}|Is], Acc, D) -> - replace(Is, [{wait,{f,label(Lbl, D)}}|Acc], D); -replace([{wait_timeout,{f,Lbl},To}|Is], Acc, D) -> - replace(Is, [{wait_timeout,{f,label(Lbl, D)},To}|Acc], D); -replace([{bif,Name,{f,Lbl},As,R}|Is], Acc, D) when Lbl =/= 0 -> - replace(Is, [{bif,Name,{f,label(Lbl, D)},As,R}|Acc], D); -replace([{gc_bif,Name,{f,Lbl},Live,As,R}|Is], Acc, D) when Lbl =/= 0 -> - replace(Is, [{gc_bif,Name,{f,label(Lbl, D)},Live,As,R}|Acc], D); -replace([{call,Ar,{f,Lbl}}|Is], Acc, D) -> - replace(Is, [{call,Ar,{f,label(Lbl,D)}}|Acc], D); -replace([{make_fun2,{f,Lbl},U1,U2,U3}|Is], Acc, D) -> - replace(Is, [{make_fun2,{f,label(Lbl, D)},U1,U2,U3}|Acc], D); -replace([{bs_init,{f,Lbl},Info,Live,Ss,Dst}|Is], Acc, D) when Lbl =/= 0 -> - replace(Is, [{bs_init,{f,label(Lbl, D)},Info,Live,Ss,Dst}|Acc], D); -replace([{bs_put,{f,Lbl},Info,Ss}|Is], Acc, D) when Lbl =/= 0 -> - replace(Is, [{bs_put,{f,label(Lbl, D)},Info,Ss}|Acc], D); -replace([{put_map=I,{f,Lbl},Op,Src,Dst,Live,List}|Is], Acc, D) - when Lbl =/= 0 -> - replace(Is, [{I,{f,label(Lbl, D)},Op,Src,Dst,Live,List}|Acc], D); -replace([{get_map_elements=I,{f,Lbl},Src,List}|Is], Acc, D) when Lbl =/= 0 -> - replace(Is, [{I,{f,label(Lbl, D)},Src,List}|Acc], D); -replace([I|Is], Acc, D) -> - replace(Is, [I|Acc], D); -replace([], Acc, _) -> Acc. - -label(Old, D) -> - case gb_trees:lookup(Old, D) of - {value,Val} -> Val; - none -> throw({error,{undefined_label,Old}}) - end. - %%% %%% Final fixup of bs_start_match2/5,bs_save2/bs_restore2 instructions for %%% new bit syntax matching (introduced in R11B). diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 4365451356..0bcec9ce19 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -71,9 +71,9 @@ %%% %%% jump L2 %%% . . . -%%% L1: %%% L2: ... %%% +%%% and all preceding uses of L1 renamed to L2. %%% If the jump is unreachable, it will be removed according to (1). %%% %%% (5) In @@ -156,41 +156,46 @@ function({function,Name,Arity,CLabel,Asm0}) -> %%% share(Is0) -> - %% We will get more sharing if we never fall through to a label. - Is = eliminate_fallthroughs(Is0, []), - share_1(Is, #{}, [], []). + Is1 = eliminate_fallthroughs(Is0, []), + Is2 = find_fixpoint(fun(Is) -> + share_1(Is, #{}, #{}, [], []) + end, Is1), + reverse(Is2). -share_1([{label,L}=Lbl|Is], Dict0, [_|_]=Seq, Acc) -> +share_1([{label,L}=Lbl|Is], Dict0, Lbls0, [_|_]=Seq, Acc) -> case maps:find(Seq, Dict0) of error -> Dict = maps:put(Seq, L, Dict0), - share_1(Is, Dict, [], [Lbl|Seq ++ Acc]); + share_1(Is, Dict, Lbls0, [], [Lbl|Seq ++ Acc]); {ok,Label} -> - share_1(Is, Dict0, [], [Lbl,{jump,{f,Label}}|Acc]) + Lbls = maps:put(L, Label, Lbls0), + share_1(Is, Dict0, Lbls, [], [Lbl,{jump,{f,Label}}|Acc]) end; -share_1([{func_info,_,_,_}=I|Is], _, [], Acc) -> - reverse(Is, [I|Acc]); -share_1([{'catch',_,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([{'try',_,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([{try_case,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([{catch_end,_}=I|Is], Dict0, Seq, Acc) -> - Dict = clean_non_sharable(Dict0), - share_1(Is, Dict, [I|Seq], Acc); -share_1([I|Is], Dict, Seq, Acc) -> +share_1([{func_info,_,_,_}|_]=Is, _, Lbls, [], Acc) when Lbls =/= #{} -> + beam_utils:replace_labels(Acc, Is, Lbls, fun(Old) -> Old end); +share_1([{func_info,_,_,_}|_]=Is, _, Lbls, [], Acc) when Lbls =:= #{} -> + reverse(Acc, Is); +share_1([{'catch',_,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{'try',_,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{try_case,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([{catch_end,_}=I|Is], Dict0, Lbls0, Seq, Acc) -> + {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0), + share_1(Is, Dict, Lbls, [I|Seq], Acc); +share_1([I|Is], Dict, Lbls, Seq, Acc) -> case is_unreachable_after(I) of false -> - share_1(Is, Dict, [I|Seq], Acc); + share_1(Is, Dict, Lbls, [I|Seq], Acc); true -> - share_1(Is, Dict, [I], Acc) + share_1(Is, Dict, Lbls, [I], Acc) end. -clean_non_sharable(Dict) -> +clean_non_sharable(Dict0, Lbls0) -> %% We are passing in or out of a 'catch' or 'try' block. Remove %% sequences that should not be shared over the boundaries of the %% block. Since the end of the sequence must match, the only @@ -198,7 +203,17 @@ clean_non_sharable(Dict) -> %% the 'catch'/'try' block is a sequence that ends with an %% instruction that causes an exception. Any sequence that causes %% an exception must contain a line/1 instruction. - maps:filter(fun(K, _V) -> sharable_with_try(K) end, Dict). + Dict1 = maps:to_list(Dict0), + Lbls1 = maps:to_list(Lbls0), + {Dict2,Lbls2} = foldl(fun({K, V}, {Dict,Lbls}) -> + case sharable_with_try(K) of + true -> + {[{K,V}|Dict],lists:keydelete(V, 2, Lbls)}; + false -> + {Dict,Lbls} + end + end, {[],Lbls1}, Dict1), + {maps:from_list(Dict2),maps:from_list(Lbls2)}. sharable_with_try([{line,_}|_]) -> %% This sequence may cause an exception and may potentially @@ -275,14 +290,15 @@ extract_seq_1(_, _) -> no. -record(st, { entry :: beam_asm:label(), %Entry label (must not be moved). - mlbl :: #{beam_asm:label() := [beam_asm:label()]}, %Moved labels. - labels :: cerl_sets:set() %Set of referenced labels. + replace :: #{beam_asm:label() := beam_asm:label()}, %Labels to replace. + labels :: cerl_sets:set(), %Set of referenced labels. + index :: beam_utils:code_index() | {lazy,[beam_utils:instruction()]} %Index built lazily only if needed }). opt(Is0, CLabel) -> find_fixpoint(fun(Is) -> Lbls = initial_labels(Is), - St = #st{entry=CLabel,mlbl=#{},labels=Lbls}, + St = #st{entry=CLabel,replace=#{},labels=Lbls,index={lazy,Is}}, opt(Is, [], St) end, Is0). @@ -292,7 +308,7 @@ find_fixpoint(OptFun, Is0) -> Is -> find_fixpoint(OptFun, Is) end. -opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc, St) -> +opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc0, St0) -> %% We have %% Test Label Ops %% jump Label @@ -301,10 +317,34 @@ opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc, St) -> case beam_utils:is_pure_test(I) of false -> %% Test is not pure; we must keep it. - opt(Is, [I|Acc], label_used(Lbl, St)); + opt(Is, [I|Acc0], label_used(Lbl, St0)); true -> %% The test is pure and its failure label is the same %% as in the jump that follows -- thus it is not needed. + %% Check if any of the previous instructions could also be eliminated. + {Acc,St} = opt_useless_loads(Acc0, L, St0), + opt(Is, Acc, St) + end; +opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) -> + %% Similar to the above, except we have a fall-through rather than jump + %% Test Label Ops + %% label Label + case beam_utils:is_pure_test(I) of + false -> + opt(Is, [I|Acc0], label_used(Lbl, St0)); + true -> + {Acc,St} = opt_useless_loads(Acc0, L, St0), + opt(Is, Acc, St) + end; +opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) -> + %% Similar to the above, except we have a fall-through rather than jump + %% Test Label Ops + %% label Label + case beam_utils:is_pure_test(I) of + false -> + opt(Is, [I|Acc0], label_used(Lbl, St0)); + true -> + {Acc,St} = opt_useless_loads(Acc0, L, St0), opt(Is, Acc, St) end; opt([{test,Test0,{f,L}=Lbl,Ops}=I|[{jump,To}|Is]=Is0], Acc, St) -> @@ -326,30 +366,16 @@ opt([{test,_,{f,_}=Lbl,_,_,_}=I|Is], Acc, St) -> opt(Is, [I|Acc], label_used(Lbl, St)); opt([{select,_,_R,Fail,Vls}=I|Is], Acc, St) -> skip_unreachable(Is, [I|Acc], label_used([Fail|Vls], St)); -opt([{label,Lbl}=I|Is], Acc, #st{mlbl=Mlbl}=St0) -> - case maps:find(Lbl, Mlbl) of - {ok,Lbls} -> - %% Essential to remove the list of labels from the dictionary, - %% since we will rescan the inserted labels. We MUST rescan. - St = St0#st{mlbl=maps:remove(Lbl, Mlbl)}, - insert_labels([Lbl|Lbls], Is, Acc, St); - error -> - opt(Is, [I|Acc], St0) - end; +opt([{label,From}=I,{label,To}|Is], Acc, #st{replace=Replace}=St) -> + opt([I|Is], Acc, St#st{replace=Replace#{To => From}}); opt([{jump,{f,_}=X}|[{label,_},{jump,X}|_]=Is], Acc, St) -> opt(Is, Acc, St); opt([{jump,{f,Lbl}}|[{label,Lbl}|_]=Is], Acc, St) -> opt(Is, Acc, St); -opt([{jump,{f,L}=Lbl}=I|Is], Acc0, #st{mlbl=Mlbl0}=St0) -> - %% All labels before this jump instruction should now be - %% moved to the location of the jump's target. - {Lbls,Acc} = collect_labels(Acc0, St0), - St = case Lbls of - [] -> St0; - [_|_] -> - Mlbl = maps_append_list(L, Lbls, Mlbl0), - St0#st{mlbl=Mlbl} - end, +opt([{jump,{f,L}=Lbl}=I|Is], Acc0, St0) -> + %% Replace all labels before this jump instruction into the + %% location of the jump's target. + {Acc,St} = collect_labels(Acc0, L, St0), skip_unreachable(Is, [I|Acc], label_used(Lbl, St)); %% Optimization: quickly handle some common instructions that don't %% have any failure labels and where is_unreachable_after(I) =:= false. @@ -369,36 +395,72 @@ opt([I|Is], Acc, #st{labels=Used0}=St0) -> true -> skip_unreachable(Is, [I|Acc], St); false -> opt(Is, [I|Acc], St) end; -opt([], Acc, #st{mlbl=Mlbl}) -> - Code = reverse(Acc), - insert_fc_labels(Code, Mlbl). - -insert_fc_labels([{label,L}=I|Is0], Mlbl) -> - case maps:find(L, Mlbl) of - error -> - [I|insert_fc_labels(Is0, Mlbl)]; - {ok,Lbls} -> - Is = [{label,Lb} || Lb <- Lbls] ++ Is0, - [I|insert_fc_labels(Is, maps:remove(L, Mlbl))] +opt([], Acc, #st{replace=Replace0}) when Replace0 =/= #{} -> + Replace = normalize_replace(maps:to_list(Replace0), Replace0, []), + beam_utils:replace_labels(Acc, [], Replace, fun(Old) -> Old end); +opt([], Acc, #st{replace=Replace}) when Replace =:= #{} -> + reverse(Acc). + +normalize_replace([{From,To0}|Rest], Replace, Acc) -> + case Replace of + #{To0 := To} -> + normalize_replace([{From,To}|Rest], Replace, Acc); + _ -> + normalize_replace(Rest, Replace, [{From,To0}|Acc]) end; -insert_fc_labels([_|_]=Is, _) -> Is. - -maps_append_list(K,Vs,M) -> - case M of - #{K:=Vs0} -> M#{K:=Vs0++Vs}; % same order as dict - _ -> M#{K => Vs} - end. +normalize_replace([], _Replace, Acc) -> + maps:from_list(Acc). + +%% After eliminating a test, it might happen, that a register was only used +%% in this test. Let's check if that was the case and if it was so, we can +%% eliminate the load into the register completely. +opt_useless_loads([{block,_}|_]=Is, L, #st{index={lazy,FIs}}=St) -> + opt_useless_loads(Is, L, St#st{index=beam_utils:index_labels(FIs)}); +opt_useless_loads([{block,Block0}|Is], L, #st{index=Index}=St) -> + case opt_useless_block_loads(Block0, L, Index) of + [] -> + opt_useless_loads(Is, L, St); + [_|_]=Block -> + {[{block,Block}|Is],St} + end; +%% After eliminating the test and useless blocks, it might happen, +%% that the previous test could also be eliminated. +%% It might be that the label was already marked as used, even if ultimately, +%% it never will be - we can't do much about it at that point, though +opt_useless_loads([{test,_,{f,L},_}=I|Is], L, St) -> + case beam_utils:is_pure_test(I) of + false -> + {[I|Is],St}; + true -> + opt_useless_loads(Is, L, St) + end; +opt_useless_loads(Is, _L, St) -> + {Is,St}. + +opt_useless_block_loads([{set,[Dst],_,_}=I|Is], L, Index) -> + BlockJump = [{block,Is},{jump,{f,L}}], + case beam_utils:is_killed(Dst, BlockJump, Index) of + true -> + %% The register is killed and not used, we can remove the load + opt_useless_block_loads(Is, L, Index); + false -> + [I|opt_useless_block_loads(Is, L, Index)] + end; +opt_useless_block_loads([I|Is], L, Index) -> + [I|opt_useless_block_loads(Is, L, Index)]; +opt_useless_block_loads([], _L, _Index) -> + []. -collect_labels(Is, #st{entry=Entry}) -> - collect_labels_1(Is, Entry, []). +collect_labels(Is, Label, #st{entry=Entry,replace=Replace} = St) -> + collect_labels_1(Is, Label, Entry, Replace, St). -collect_labels_1([{label,Entry}|_]=Is, Entry, Acc) -> +collect_labels_1([{label,Entry}|_]=Is, _Label, Entry, Acc, St) -> %% Never move the entry label. - {Acc,Is}; -collect_labels_1([{label,L}|Is], Entry, Acc) -> - collect_labels_1(Is, Entry, [L|Acc]); -collect_labels_1(Is, _Entry, Acc) -> - {Acc,Is}. + {Is,St#st{replace=Acc}}; +collect_labels_1([{label,L}|Is], Label, Entry, Acc, St) -> + collect_labels_1(Is, Label, Entry, Acc#{L => Label}, St); +collect_labels_1(Is, _Label, _Entry, Acc, St) -> + {Is,St#st{replace=Acc}}. %% label_defined(Is, Label) -> true | false. %% Test whether the label Label is defined at the start of the instruction @@ -418,13 +480,6 @@ invert_test(is_eq_exact) -> is_ne_exact; invert_test(is_ne_exact) -> is_eq_exact; invert_test(_) -> not_possible. -insert_labels([L|Ls], Is, [{jump,{f,L}}|Acc], St) -> - insert_labels(Ls, [{label,L}|Is], Acc, St); -insert_labels([L|Ls], Is, Acc, St) -> - insert_labels(Ls, [{label,L}|Is], Acc, St); -insert_labels([], Is, Acc, St) -> - opt(Is, Acc, St). - %% skip_unreachable([Instruction], St). %% Remove all instructions (including definitions of labels %% that have not been referenced yet) up to the next diff --git a/lib/compiler/src/beam_peep.erl b/lib/compiler/src/beam_peep.erl index 6df5c02334..9436c20b36 100644 --- a/lib/compiler/src/beam_peep.erl +++ b/lib/compiler/src/beam_peep.erl @@ -89,15 +89,37 @@ peep([{gc_bif,_,_,_,_,Dst}=I|Is], SeenTests0, Acc) -> peep([{jump,{f,L}},{label,L}=I|Is], _, Acc) -> %% Sometimes beam_jump has missed this optimization. peep(Is, gb_sets:empty(), [I|Acc]); -peep([{select,Op,R,F,Vls0}|Is], _, Acc) -> +peep([{select,Op,R,F,Vls0}|Is], SeenTests0, Acc0) -> case prune_redundant_values(Vls0, F) of [] -> %% No values left. Must convert to plain jump. I = {jump,F}, - peep(Is, gb_sets:empty(), [I|Acc]); + peep([I|Is], gb_sets:empty(), Acc0); + [{atom,_}=Value,Lbl] when Op =:= select_val -> + %% Single value left. Convert to regular test and pop redundant tests. + Is1 = [{test,is_eq_exact,F,[R,Value]},{jump,Lbl}|Is], + case Acc0 of + [{test,is_atom,F,[R]}|Acc] -> + peep(Is1, SeenTests0, Acc); + _ -> + peep(Is1, SeenTests0, Acc0) + end; + [{integer,_}=Value,Lbl] when Op =:= select_val -> + %% Single value left. Convert to regular test and pop redundant tests. + Is1 = [{test,is_eq_exact,F,[R,Value]},{jump,Lbl}|Is], + case Acc0 of + [{test,is_integer,F,[R]}|Acc] -> + peep(Is1, SeenTests0, Acc); + _ -> + peep(Is1, SeenTests0, Acc0) + end; + [Arity,Lbl] when Op =:= select_tuple_arity -> + %% Single value left. Convert to regular test + Is1 = [{test,test_arity,F,[R,Arity]},{jump,Lbl}|Is], + peep(Is1, SeenTests0, Acc0); [_|_]=Vls -> I = {select,Op,R,F,Vls}, - peep(Is, gb_sets:empty(), [I|Acc]) + peep(Is, gb_sets:empty(), [I|Acc0]) end; peep([{test,Op,_,Ops}=I|Is], SeenTests0, Acc) -> case beam_utils:is_pure_test(I) of diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl index cc6e54ca16..52ed1c7ca0 100644 --- a/lib/compiler/src/beam_utils.erl +++ b/lib/compiler/src/beam_utils.erl @@ -23,14 +23,14 @@ -module(beam_utils). -export([is_killed_block/2,is_killed/3,is_killed_at/3, is_not_used/3, - empty_label_index/0,index_label/3,index_labels/1, + empty_label_index/0,index_label/3,index_labels/1,replace_labels/4, code_at/2,bif_to_test/3,is_pure_test/1, live_opt/1,delete_live_annos/1,combine_heap_needs/2, split_even/1]). -export_type([code_index/0,module_code/0,instruction/0]). --import(lists, [member/2,sort/1,reverse/1,splitwith/2]). +-import(lists, [map/2,member/2,sort/1,reverse/1,splitwith/2]). %% instruction() describes all instructions that are used during optimzation %% (from beam_a to beam_z). @@ -160,6 +160,18 @@ index_label(Lbl, Is0, Acc) -> code_at(L, Ll) -> gb_trees:get(L, Ll). +%% replace_labels(FunctionIs, Tail, ReplaceDb, Fallback) -> FunctionIs. +%% Replace all labels in instructions according to the ReplaceDb. +%% If label is not found the Fallback is called with the label to +%% produce a new one. + +-spec replace_labels([instruction()], + [instruction()], + #{beam_asm:label() => beam_asm:label()}, + fun((beam_asm:label()) -> term())) -> [instruction()]. +replace_labels(Is, Acc, D, Fb) -> + replace_labels_1(Is, Acc, D, Fb). + %% bif_to_test(Bif, [Op], Fail) -> {test,Test,Fail,[Op]} %% Convert a BIF to a test. Fail if not possible. @@ -643,6 +655,58 @@ index_labels_1([], Acc) -> gb_trees:from_orddict(sort(Acc)). drop_labels([{label,_}|Is]) -> drop_labels(Is); drop_labels(Is) -> Is. + +replace_labels_1([{test,Test,{f,Lbl},Ops}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{test,Test,{f,label(Lbl, D, Fb)},Ops}|Acc], D, Fb); +replace_labels_1([{test,Test,{f,Lbl},Live,Ops,Dst}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{test,Test,{f,label(Lbl, D, Fb)},Live,Ops,Dst}|Acc], D, Fb); +replace_labels_1([{select,I,R,{f,Fail0},Vls0}|Is], Acc, D, Fb) -> + Vls = map(fun ({f,L}) -> {f,label(L, D, Fb)}; + (Other) -> Other + end, Vls0), + Fail = label(Fail0, D, Fb), + replace_labels_1(Is, [{select,I,R,{f,Fail},Vls}|Acc], D, Fb); +replace_labels_1([{'try',R,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{'try',R,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{'catch',R,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{'catch',R,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{jump,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{jump,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{loop_rec,{f,Lbl},R}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{loop_rec,{f,label(Lbl, D, Fb)},R}|Acc], D, Fb); +replace_labels_1([{loop_rec_end,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{loop_rec_end,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{wait,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{wait,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{wait_timeout,{f,Lbl},To}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{wait_timeout,{f,label(Lbl, D, Fb)},To}|Acc], D, Fb); +replace_labels_1([{bif,Name,{f,Lbl},As,R}|Is], Acc, D, Fb) when Lbl =/= 0 -> + replace_labels_1(Is, [{bif,Name,{f,label(Lbl, D, Fb)},As,R}|Acc], D, Fb); +replace_labels_1([{gc_bif,Name,{f,Lbl},Live,As,R}|Is], Acc, D, Fb) when Lbl =/= 0 -> + replace_labels_1(Is, [{gc_bif,Name,{f,label(Lbl, D, Fb)},Live,As,R}|Acc], D, Fb); +replace_labels_1([{call,Ar,{f,Lbl}}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{call,Ar,{f,label(Lbl, D, Fb)}}|Acc], D, Fb); +replace_labels_1([{make_fun2,{f,Lbl},U1,U2,U3}|Is], Acc, D, Fb) -> + replace_labels_1(Is, [{make_fun2,{f,label(Lbl, D, Fb)},U1,U2,U3}|Acc], D, Fb); +replace_labels_1([{bs_init,{f,Lbl},Info,Live,Ss,Dst}|Is], Acc, D, Fb) when Lbl =/= 0 -> + replace_labels_1(Is, [{bs_init,{f,label(Lbl, D, Fb)},Info,Live,Ss,Dst}|Acc], D, Fb); +replace_labels_1([{bs_put,{f,Lbl},Info,Ss}|Is], Acc, D, Fb) when Lbl =/= 0 -> + replace_labels_1(Is, [{bs_put,{f,label(Lbl, D, Fb)},Info,Ss}|Acc], D, Fb); +replace_labels_1([{put_map=I,{f,Lbl},Op,Src,Dst,Live,List}|Is], Acc, D, Fb) + when Lbl =/= 0 -> + replace_labels_1(Is, [{I,{f,label(Lbl, D, Fb)},Op,Src,Dst,Live,List}|Acc], D, Fb); +replace_labels_1([{get_map_elements=I,{f,Lbl},Src,List}|Is], Acc, D, Fb) when Lbl =/= 0 -> + replace_labels_1(Is, [{I,{f,label(Lbl, D, Fb)},Src,List}|Acc], D, Fb); +replace_labels_1([I|Is], Acc, D, Fb) -> + replace_labels_1(Is, [I|Acc], D, Fb); +replace_labels_1([], Acc, _, _) -> Acc. + +label(Old, D, Fb) -> + case D of + #{Old := New} -> New; + _ -> Fb(Old) + end. + %% Help functions for combine_heap_needs. combine_alloc_lists(Al1, Al2) -> @@ -789,39 +853,48 @@ live_opt([{recv_mark,_}=I|Is], Regs, D, Acc) -> live_opt([], _, _, Acc) -> Acc. -live_opt_block([{set,Ds,Ss,Op}=I0|Is], Regs0, D, Acc) -> +live_opt_block([{set,Ds,Ss,Op0}|Is], Regs0, D, Acc) -> Regs1 = x_live(Ss, x_dead(Ds, Regs0)), - {I,Regs} = case Op of - {alloc,Live0,Alloc} -> - %% The life-time analysis used by the code generator - %% is sometimes too conservative, so it may be - %% possible to lower the number of live registers - %% based on the exact liveness information. - %% The main benefit is that more optimizations that - %% depend on liveness information (such as the - %% beam_bool and beam_dead passes) may be applied. - Live = live_regs(Regs1), - true = Live =< Live0, %Assertion. - I1 = {set,Ds,Ss,{alloc,Live,Alloc}}, - {I1,live_call(Live)}; - _ -> - {I0,Regs1} - end, + {Op, Regs} = live_opt_block_op(Op0, Regs1, D), + I = {set, Ds, Ss, Op}, + case Ds of - [{x,X}] -> - case (not is_live(X, Regs0)) andalso Op =:= move of - true -> - live_opt_block(Is, Regs0, D, Acc); - false -> - live_opt_block(Is, Regs, D, [I|Acc]) - end; - _ -> - live_opt_block(Is, Regs, D, [I|Acc]) + [{x,X}] -> + case (not is_live(X, Regs0)) andalso Op =:= move of + true -> + live_opt_block(Is, Regs0, D, Acc); + false -> + live_opt_block(Is, Regs, D, [I|Acc]) + end; + _ -> + live_opt_block(Is, Regs, D, [I|Acc]) end; live_opt_block([{'%live',_,_}|Is], Regs, D, Acc) -> live_opt_block(Is, Regs, D, Acc); live_opt_block([], Regs, _, Acc) -> {Acc,Regs}. +live_opt_block_op({alloc,Live0,AllocOp}, Regs0, D) -> + Regs = + case AllocOp of + {Kind, _N, Fail} when Kind =:= gc_bif; Kind =:= put_map -> + live_join_label(Fail, D, Regs0); + _ -> + Regs0 + end, + + %% The life-time analysis used by the code generator is sometimes too + %% conservative, so it may be possible to lower the number of live + %% registers based on the exact liveness information. The main benefit is + %% that more optimizations that depend on liveness information (such as the + %% beam_bool and beam_dead passes) may be applied. + Live = live_regs(Regs), + true = Live =< Live0, + {{alloc,Live,AllocOp}, live_call(Live)}; +live_opt_block_op({bif,_N,Fail} = Op, Regs, D) -> + {Op, live_join_label(Fail, D, Regs)}; +live_opt_block_op(Op, Regs, _D) -> + {Op, Regs}. + live_join_labels([{f,L}|T], D, Regs0) when L =/= 0 -> Regs = gb_trees:get(L, D) bor Regs0, live_join_labels(T, D, Regs); diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index f726625510..00901077d3 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -928,9 +928,9 @@ verify_call_match_context(Lbl, Ctx, #vst{ft=Ft}) -> error({unsuitable_bs_start_match2,I}) end. -allocate(Zero, Stk, Heap, Live, #vst{current=#st{numy=none}=St}=Vst0) -> +allocate(Zero, Stk, Heap, Live, #vst{current=#st{numy=none}}=Vst0) -> verify_live(Live, Vst0), - Vst = prune_x_regs(Live, Vst0), + Vst = #vst{current=St} = prune_x_regs(Live, Vst0), Ys = init_regs(Stk, case Zero of true -> initialized; false -> uninitialized @@ -1430,13 +1430,13 @@ merge_types(bool, {atom,A}) -> merge_bool(A); merge_types({atom,A}, bool) -> merge_bool(A); -merge_types(#ms{id=Id,valid=B0,slots=Slots}=M, - #ms{id=Id,valid=B1,slots=Slots}) -> - M#ms{valid=B0 bor B1,slots=Slots}; -merge_types(#ms{}=M, _) -> - M; -merge_types(_, #ms{}=M) -> - M; +merge_types(#ms{id=Id1,valid=B0,slots=Slots}, + #ms{id=Id2,valid=B1,slots=Slots}) -> + Id = if + Id1 =:= Id2 -> Id1; + true -> make_ref() + end, + #ms{id=Id,valid=B0 band B1,slots=Slots}; merge_types(T1, T2) when T1 =/= T2 -> %% Too different. All we know is that the type is a 'term'. term. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index aa2d224bb4..ec7e7aed14 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -706,14 +706,16 @@ core_passes() -> [{unless,no_copt, [{core_old_inliner,fun test_old_inliner/1,fun core_old_inliner/2}, {iff,doldinline,{listing,"oldinline"}}, - {pass,sys_core_fold}, + {unless,no_fold,{pass,sys_core_fold}}, {iff,dcorefold,{listing,"corefold"}}, {core_inline_module,fun test_core_inliner/1,fun core_inline_module/2}, {iff,dinline,{listing,"inline"}}, {core_fold_after_inlining,fun test_any_inliner/1, fun core_fold_module_after_inlining/2}, + {iff,dcopt,{listing,"copt"}}, + {unless,no_alias,{pass,sys_core_alias}}, + {iff,dalias,{listing,"core_alias"}}, ?pass(core_transforms)]}, - {iff,dcopt,{listing,"copt"}}, {iff,'to_core',{done,"core"}}]} | kernel_passes()]. @@ -1921,6 +1923,7 @@ pre_load() -> erl_lint, erl_parse, erl_scan, + sys_core_alias, sys_core_bsm, sys_core_dsetel, sys_core_fold, diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index 3139d68902..703cf1d1b8 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -58,6 +58,7 @@ core_lib, erl_bifs, rec_env, + sys_core_alias, sys_core_bsm, sys_core_dsetel, sys_core_fold, diff --git a/lib/compiler/src/sys_core_alias.erl b/lib/compiler/src/sys_core_alias.erl new file mode 100644 index 0000000000..63e2f7488e --- /dev/null +++ b/lib/compiler/src/sys_core_alias.erl @@ -0,0 +1,308 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% Purpose : Replace values by aliases from patterns optimisation for Core + +%% Replace expressions by aliases from patterns. For example: +%% +%% example({ok, Val}) -> +%% {ok, Val}. +%% +%% will become: +%% +%% example({ok, Val} = Tuple) -> +%% Tuple. +%% +%% Currently this pass aliases tuple and cons nodes made of literals, +%% variables and other cons. The tuple/cons may appear anywhere in the +%% pattern and it will be aliased if used later on. +%% +%% Notice a tuple/cons made only of literals is not aliased as it may +%% be part of the literal pool. + +-module(sys_core_alias). + +-export([module/2]). + +-include("core_parse.hrl"). + +-define(NOTSET, 0). + +-record(sub, {p=#{} :: #{term() => ?NOTSET | atom()}, %% Found pattern substitutions + v=cerl_sets:new() :: cerl_sets:set(cerl:var_name()), %% Variables used by patterns + t=undefined :: term()}). %% Temporary information from pre to post + +-type sub() :: #sub{}. + +-spec module(cerl:c_module(), [compile:option()]) -> + {'ok',cerl:c_module(),[]}. + +module(#c_module{defs=Ds0}=Mod, _Opts) -> + Ds1 = [def(D) || D <- Ds0], + {ok,Mod#c_module{defs=Ds1},[]}. + +def({#c_var{name={F,Arity}}=Name,B0}) -> + try + put(new_var_num, 0), + {B1,_} = cerl_trees:mapfold(fun pre/2, fun post/2, sub_new(undefined), B0), + erase(new_var_num), + {Name,B1} + catch + Class:Error -> + Stack = erlang:get_stacktrace(), + io:fwrite("Function: ~w/~w\n", [F,Arity]), + erlang:raise(Class, Error, Stack) + end. + +pre(#c_let{vars=Vars}=Node, Sub) -> + {Node,sub_fold(get_variables(Vars), Sub)}; + +pre(#c_fun{vars=Vars}=Node, Sub) -> + {Node,sub_fold(get_variables(Vars), Sub)}; + +pre(#c_clause{pats=Pats}=Node, Sub0) -> + VarNames = get_variables(Pats), + Sub1 = sub_fold(VarNames, Sub0), + Keys = get_pattern_keys(Pats), + Sub2 = sub_add_keys(Keys, Sub1), + + #sub{v=SubNames,t=Temp} = Sub2, + Sub3 = Sub2#sub{v=merge_variables(VarNames, SubNames), + t={clause,Pats,Keys,SubNames,Temp}}, + + {Node#c_clause{pats=[]},Sub3}; + +pre(Node, Sub0) -> + %% We cache only tuples and cons. + case cerl:is_data(Node) andalso not cerl:is_literal(Node) of + false -> + {Node,Sub0}; + true -> + Kind = cerl:data_type(Node), + Es = cerl:data_es(Node), + case sub_cache_nodes(Kind, Es, Sub0) of + {Name,Sub1} -> + {cerl:ann_c_var(cerl:get_ann(Node), Name),Sub1}; + error -> + {Node,Sub0} + end + end. + +post(#c_let{}=Node, Sub) -> + {Node,sub_unfold(Sub)}; + +post(#c_fun{}=Node, Sub) -> + {Node,sub_unfold(Sub)}; + +post(#c_clause{}=Node, #sub{t={clause,Pats0,Keys,V,T}}=Sub0) -> + {Sub1,PostKeys} = sub_take_keys(Keys, Sub0), + Pats1 = put_pattern_keys(Pats0, PostKeys), + Sub2 = sub_unfold(Sub1#sub{v=V,t=T}), + {Node#c_clause{pats=Pats1},Sub2}; + +post(Node, Sub) -> + {Node,Sub}. + +%% sub_new/1 +%% sub_add_keys/2 +%% sub_take_keys/3 +%% sub_cache_nodes/3 +%% +%% Manages the substitutions record. + +%% Builds a new sub. +-spec sub_new(term()) -> sub(). +sub_new(Temp) -> + #sub{t=Temp}. + +%% Folds the sub into a new one if the variables in nodes are not disjoint +sub_fold(VarNames, #sub{v=SubNames}=Sub) -> + case is_disjoint_variables(VarNames, SubNames) of + true -> Sub#sub{t={temp,Sub#sub.t}}; + false -> sub_new({sub,Sub}) + end. + +%% Unfolds the sub in case one was folded in the previous step +sub_unfold(#sub{t={temp,Temp}}=Sub) -> + Sub#sub{t=Temp}; +sub_unfold(#sub{t={sub,Sub}}) -> + Sub. + +%% Adds the keys extracted from patterns to the state. +-spec sub_add_keys([term()], sub()) -> sub(). +sub_add_keys(Keys, #sub{p=Pat0}=Sub) -> + Pat1 = + lists:foldl(fun(Key, Acc) -> + false = maps:is_key(Key, Acc), %Assertion. + maps:put(Key, ?NOTSET, Acc) + end, Pat0, Keys), + Sub#sub{p=Pat1}. + +%% Take the keys from the map taking into account the keys +%% that have changed as those must become aliases in the pattern. +-spec sub_take_keys([term()], sub()) -> {sub(), [{term(), atom()}]}. +sub_take_keys(Keys, #sub{p=Pat0}=Sub) -> + {Pat1,Acc} = sub_take_keys(Keys, Pat0, []), + {Sub#sub{p=Pat1},Acc}. + +sub_take_keys([K|T], Sub0, Acc) -> + case maps:take(K, Sub0) of + {?NOTSET,Sub1} -> + sub_take_keys(T, Sub1, Acc); + {Name,Sub1} -> + sub_take_keys(T, Sub1, [{K,Name}|Acc]) + end; +sub_take_keys([], Sub, Acc) -> + {Sub,Acc}. + +%% Check if the node can be cached based on the state information. +%% If it can be cached and it does not have an alias for it, we +%% build one. +-spec sub_cache_nodes(atom(), [cerl:cerl()], sub()) -> {atom(), sub()} | error. +sub_cache_nodes(Kind, Nodes, #sub{p=Pat}=Sub) -> + case nodes_to_key(Kind, Nodes) of + {ok, Key} -> + case Pat of + #{Key := ?NOTSET} -> + new_var_name(Key, Sub); + #{Key := Name} -> + {Name,Sub}; + #{} -> + error + end; + error -> + error + end. + +new_var_name(Key, #sub{p=Pat}=Sub) -> + Counter = get(new_var_num), + Name = list_to_atom("@r" ++ integer_to_list(Counter)), + put(new_var_num, Counter + 1), + {Name,Sub#sub{p=maps:put(Key, Name, Pat)}}. + +%% get_variables/1 +%% is_disjoint_variables/2 +%% merge_variables/2 + +get_variables(NodesList) -> + cerl_sets:from_list([Var || Node <- NodesList, Var <- cerl_trees:variables(Node)]). + +is_disjoint_variables(Vars1, Vars2) -> + cerl_sets:is_disjoint(Vars1, Vars2). + +merge_variables(Vars1, Vars2) -> + cerl_sets:union(Vars1, Vars2). + +%% get_pattern_keys/2 +%% put_pattern_keys/2 +%% +%% Gets keys from patterns or add them as aliases. + +get_pattern_keys(Patterns) -> + lists:foldl(fun get_pattern_keys/2, [], Patterns). + +get_pattern_keys(#c_tuple{es=Es}, Acc0) -> + Acc1 = accumulate_pattern_keys(tuple, Es, Acc0), + lists:foldl(fun get_pattern_keys/2, Acc1, Es); +get_pattern_keys(#c_cons{hd=Hd,tl=Tl}, Acc0) -> + Acc1 = accumulate_pattern_keys(cons, [Hd, Tl], Acc0), + get_pattern_keys(Tl, get_pattern_keys(Hd, Acc1)); +get_pattern_keys(#c_alias{pat=Pat}, Acc0) -> + get_pattern_keys(Pat, Acc0); +get_pattern_keys(#c_map{es=Es}, Acc0) -> + lists:foldl(fun get_pattern_keys/2, Acc0, Es); +get_pattern_keys(#c_map_pair{val=Val}, Acc0) -> + get_pattern_keys(Val, Acc0); +get_pattern_keys(_, Acc) -> + Acc. + +accumulate_pattern_keys(Kind, Nodes, Acc) -> + case nodes_to_key(Kind, Nodes) of + {ok,Key} -> [Key|Acc]; + error -> Acc + end. + +put_pattern_keys(Patterns, []) -> + Patterns; +put_pattern_keys(Patterns, Keys) -> + {NewPatterns,Map} = + lists:mapfoldl(fun alias_pattern_keys/2, maps:from_list(Keys), Patterns), + %% Check all aliases have been consumed from the map. + 0 = map_size(Map), + NewPatterns. + +alias_pattern_keys(#c_tuple{anno=Anno,es=Es0}=Node, Acc0) -> + {Es1,Acc1} = lists:mapfoldl(fun alias_pattern_keys/2, Acc0, Es0), + nodes_to_alias(tuple, Es0, Anno, Node#c_tuple{es=Es1}, Acc1); +alias_pattern_keys(#c_cons{anno=Anno,hd=Hd0,tl=Tl0}=Node, Acc0) -> + {Hd1,Acc1} = alias_pattern_keys(Hd0, Acc0), + {Tl1,Acc2} = alias_pattern_keys(Tl0, Acc1), + nodes_to_alias(cons, [Hd0, Tl0], Anno, Node#c_cons{hd=Hd1,tl=Tl1}, Acc2); +alias_pattern_keys(#c_alias{pat=Pat0}=Node, Acc0) -> + {Pat1,Acc1} = alias_pattern_keys(Pat0, Acc0), + {Node#c_alias{pat=Pat1}, Acc1}; +alias_pattern_keys(#c_map{es=Es0}=Node, Acc0) -> + {Es1,Acc1} = lists:mapfoldl(fun alias_pattern_keys/2, Acc0, Es0), + {Node#c_map{es=Es1}, Acc1}; +alias_pattern_keys(#c_map_pair{val=Val0}=Node, Acc0) -> + {Val1,Acc1} = alias_pattern_keys(Val0, Acc0), + {Node#c_map_pair{val=Val1}, Acc1}; +alias_pattern_keys(Pattern, Acc) -> + {Pattern,Acc}. + +%% Check if a node must become an alias because +%% its pattern was used later on as an expression. +nodes_to_alias(Kind, Inner, Anno, Node, Keys0) -> + case nodes_to_key(Kind, Inner) of + {ok,Key} -> + case maps:take(Key, Keys0) of + {Name,Keys1} -> + Var = cerl:ann_c_var(Anno, Name), + {cerl:ann_c_alias(Anno, Var, Node), Keys1}; + error -> + {Node,Keys0} + end; + error -> + {Node,Keys0} + end. + +%% Builds the key used to check if a value can be +%% replaced by an alias. It considers literals, +%% aliases, variables, tuples and cons recursively. +nodes_to_key(Kind, Nodes) -> + nodes_to_key(Nodes, [], Kind). + +nodes_to_key([#c_alias{var=Var}|T], Acc, Kind) -> + nodes_to_key([Var|T], Acc, Kind); +nodes_to_key([#c_var{name=Name}|T], Acc, Kind) -> + nodes_to_key(T, [[var,Name]|Acc], Kind); +nodes_to_key([Node|T], Acc0, Kind) -> + case cerl:is_data(Node) of + false -> + error; + true -> + case nodes_to_key(cerl:data_es(Node), [], cerl:data_type(Node)) of + {ok,Key} -> + nodes_to_key(T, [Key|Acc0], Kind); + error -> + error + end + end; +nodes_to_key([], Acc, Kind) -> + {ok,[Kind|Acc]}. diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index e0cd6da06f..f3f315935a 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -395,10 +395,10 @@ expr(#c_receive{clauses=Cs0,timeout=T0,action=A0}=Recv, Ctxt, Sub) -> expr(#c_apply{anno=Anno,op=Op0,args=As0}=App, _, Sub) -> Op1 = expr(Op0, value, Sub), As1 = expr_list(As0, value, Sub), - case Op1 of - #c_var{} -> + case cerl:is_data(Op1) of + false -> App#c_apply{op=Op1,args=As1}; - _ -> + true -> add_warning(App, invalid_call), Err = #c_call{anno=Anno, module=#c_literal{val=erlang}, @@ -2422,16 +2422,10 @@ move_let_into_expr(#c_let{vars=InnerVs0,body=InnerBody0}=Inner, Outer#c_let{vars=OuterVs,arg=Arg, body=Inner#c_let{vars=InnerVs,arg=OuterBody,body=InnerBody}}; move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, - #c_case{arg=Cexpr0,clauses=[Ca0,Cb0|Cs]}=Case, Sub0) -> - %% Test if there are no more clauses than Ca0 and Cb0, or if - %% Cb0 is guaranteed to match. - TwoClauses = Cs =:= [] orelse - case Cb0 of - #c_clause{pats=[#c_var{}],guard=#c_literal{val=true}} -> true; - _ -> false - end, - case {TwoClauses,is_failing_clause(Ca0),is_failing_clause(Cb0)} of - {true,false,true} -> + #c_case{arg=Cexpr0,clauses=[Ca0|Cs0]}=Case, Sub0) -> + case not is_failing_clause(Ca0) andalso + are_all_failing_clauses(Cs0) of + true -> %% let <Lvars> = case <Case-expr> of %% <Cpats> -> <Clause-body>; %% <OtherCpats> -> erlang:error(...) @@ -2467,8 +2461,8 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, body=Lbody}, Ca = Ca0#c_clause{pats=CaPats,guard=G,body=B}, - Cb = clause(Cb0, Cexpr, value, Sub0), - Case#c_case{arg=Cexpr,clauses=[Ca,Cb]} + Cs = [clause(C, Cexpr, value, Sub0) || C <- Cs0], + Case#c_case{arg=Cexpr,clauses=[Ca|Cs]} catch nomatch -> %% This is not a defeat. The code will eventually @@ -2476,7 +2470,7 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, %% optimizations done in this module. impossible end; - {_,_,_} -> impossible + false -> impossible end; move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, #c_seq{arg=Sarg0,body=Sbody0}=Seq, Sub0) -> @@ -2499,6 +2493,9 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, body=Lbody}}; move_let_into_expr(_Let, _Expr, _Sub) -> impossible. +are_all_failing_clauses(Cs) -> + all(fun is_failing_clause/1, Cs). + is_failing_clause(#c_clause{body=B}) -> will_fail(B). diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index 47c1567f10..0ae3289103 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -884,12 +884,19 @@ select_extract_bin([{var,Hd}], Size, Unit, binary, Flags, Vf, %% calculcated by v3_life is too conservative to be useful for this purpose.) %% 'true' means that the code that follows will definitely not use the context %% again (because it is a block, not guard or matching code); 'false' that we -%% are not sure (there is either a guard, or more matching, either which may -%% reference the context again). - -is_context_unused(#l{ke=Ke}) -> is_context_unused(Ke); -is_context_unused({block,_}) -> true; -is_context_unused(_) -> false. +%% are not sure (there could be more matching). + +is_context_unused(#l{ke=Ke}) -> + is_context_unused(Ke); +is_context_unused({alt,_First,Then}) -> + %% {alt,First,Then} can be used for different purposes. If the Then part + %% is a block, it means that matching has finished and is used for a guard + %% to choose between the matched clauses. + is_context_unused(Then); +is_context_unused({block,_}) -> + true; +is_context_unused(_) -> + false. select_bin_end(#l{ke={val_clause,{bin_end,Ctx},B}}, Ivar, Tf, Bef, St0) -> diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl index ae650546e5..20cb3343fb 100644 --- a/lib/compiler/src/v3_core.erl +++ b/lib/compiler/src/v3_core.erl @@ -2505,8 +2505,46 @@ cexpr(#ifun{anno=#a{us=Us0}=A0,name={named,Name},fc=#iclause{pats=Ps}}=Fun0, end; cexpr(#iapply{anno=A,op=Op,args=Args}, _As, St) -> {#c_apply{anno=A#a.anno,op=Op,args=Args},[],A#a.us,St}; -cexpr(#icall{anno=A,module=Mod,name=Name,args=Args}, _As, St) -> - {#c_call{anno=A#a.anno,module=Mod,name=Name,args=Args},[],A#a.us,St}; +cexpr(#icall{anno=A,module=Mod,name=Name,args=Args}, _As, St0) -> + Anno = A#a.anno, + case (not cerl:is_c_atom(Mod)) andalso member(tuple_calls, St0#core.opts) of + true -> + GenAnno = [compiler_generated|Anno], + + %% Generate the clause that matches on the tuple + {TupleVar,St1} = new_var(GenAnno, St0), + {TupleSizeVar, St2} = new_var(GenAnno, St1), + {TupleModVar, St3} = new_var(GenAnno, St2), + {TupleArgsVar, St4} = new_var(GenAnno, St3), + TryVar = cerl:c_var('Try'), + + TupleGuardExpr = + cerl:c_let([TupleSizeVar], + c_call_erl(tuple_size, [TupleVar]), + c_call_erl('>', [TupleSizeVar, cerl:c_int(0)])), + + TupleGuard = + cerl:c_try(TupleGuardExpr, [TryVar], TryVar, + [cerl:c_var('T'),cerl:c_var('R')], cerl:c_atom(false)), + + TupleApply = + cerl:c_let([TupleModVar], + c_call_erl(element, [cerl:c_int(1),TupleVar]), + cerl:c_let([TupleArgsVar], + cerl:make_list(Args ++ [TupleVar]), + c_call_erl(apply, [TupleModVar,Name,TupleArgsVar]))), + + TupleClause = cerl:ann_c_clause(GenAnno, [TupleVar], TupleGuard, TupleApply), + + %% Generate the fallback clause + {OtherVar,St5} = new_var(GenAnno, St4), + OtherApply = cerl:ann_c_call(GenAnno, OtherVar, Name, Args), + OtherClause = cerl:ann_c_clause(GenAnno, [OtherVar], OtherApply), + + {cerl:ann_c_case(GenAnno, Mod, [TupleClause,OtherClause]),[],A#a.us,St5}; + false -> + {#c_call{anno=Anno,module=Mod,name=Name,args=Args},[],A#a.us,St0} + end; cexpr(#iprimop{anno=A,name=Name,args=Args}, _As, St) -> {#c_primop{anno=A#a.anno,name=Name,args=Args},[],A#a.us,St}; cexpr(#iprotect{anno=A,body=Es}, _As, St0) -> @@ -2536,6 +2574,9 @@ cfun(#ifun{anno=A,id=Id,vars=Args,clauses=Lcs,fc=Lfc}, _As, St0) -> clauses=Ccs ++ [Cfc]}}, [],A#a.us,St2}. +c_call_erl(Fun, Args) -> + cerl:c_call(cerl:c_atom(erlang), cerl:c_atom(Fun), Args). + %% lit_vars(Literal) -> [Var]. lit_vars(Lit) -> lit_vars(Lit, []). diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index 63763f31b2..da5d207db9 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -22,6 +22,7 @@ MODULES= \ bs_construct_SUITE \ bs_match_SUITE \ bs_utf_SUITE \ + core_alias_SUITE \ core_fold_SUITE \ compile_SUITE \ compilation_SUITE \ diff --git a/lib/compiler/test/beam_utils_SUITE.erl b/lib/compiler/test/beam_utils_SUITE.erl index a3f1bb93fe..710cb050d4 100644 --- a/lib/compiler/test/beam_utils_SUITE.erl +++ b/lib/compiler/test/beam_utils_SUITE.erl @@ -260,6 +260,14 @@ otp_8949_b(A, B) -> liveopt(_Config) -> F = liveopt_fun(42, pebkac, user), void = F(42, #alarmInfo{type=sctp,cause=pebkac,origin=user}), + + + A = {#alarmInfo{cause = {abc, def}}, ghi}, + A = liveopt_guard_bif(A), + + B = {#alarmInfo{cause = {abc}}, def}, + {#alarmInfo{cause = {{abc}}}, def} = liveopt_guard_bif(B), + ok. liveopt_fun(Peer, Cause, Origin) -> @@ -271,6 +279,15 @@ liveopt_fun(Peer, Cause, Origin) -> void end. +liveopt_guard_bif({#alarmInfo{cause=F}=R, X}=A) -> + %% ERIERL-48 + if + is_tuple(F), tuple_size(F) == 2 -> A; + true -> + R2 = R#alarmInfo{cause={F}}, + {R2,X} + end. + %% Thanks to QuickCheck. coverage(_Config) -> 42+7 = merchant([[],7,false]), diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 0ec05456ec..1da7f68dab 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -39,7 +39,7 @@ match_string_opt/1,select_on_integer/1, map_and_binary/1,unsafe_branch_caching/1, bad_literals/1,good_literals/1,constant_propagation/1, - parse_xml/1,get_payload/1]). + parse_xml/1,get_payload/1,escape/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -71,7 +71,7 @@ groups() -> match_string_opt,select_on_integer, map_and_binary,unsafe_branch_caching, bad_literals,good_literals,constant_propagation,parse_xml, - get_payload]}]. + get_payload,escape]}]. init_per_suite(Config) -> @@ -1524,6 +1524,21 @@ do_get_payload(ExtHdr) -> <<_:13,_:35>> = ExtHdr#ext_header.ext_hdr_opts, ExtHdrOptions. +escape(_Config) -> + 0 = escape(<<>>, 0), + 1 = escape(<<128>>, 0), + 2 = escape(<<128,255>>, 0), + 42 = escape(<<42>>, 0), + 50 = escape(<<42,8>>, 0), + ok. + +escape(<<Byte, Rest/bits>>, Pos) when Byte >= 127 -> + escape(Rest, Pos + 1); +escape(<<Byte, Rest/bits>>, Pos) -> + escape(Rest, Pos + Byte); +escape(<<_Rest/bits>>, Pos) -> + Pos. + check(F, R) -> R = F(). diff --git a/lib/compiler/test/compilation_SUITE_data/opt_crash.erl b/lib/compiler/test/compilation_SUITE_data/opt_crash.erl index f1607cca68..c65ec31593 100644 --- a/lib/compiler/test/compilation_SUITE_data/opt_crash.erl +++ b/lib/compiler/test/compilation_SUITE_data/opt_crash.erl @@ -33,7 +33,7 @@ test() -> {userinfo,nil}, fun() -> nil end}, nil}, - {'query',nil}}}, + {query,nil}}}, {absoluteURI, {scheme,_}, @@ -43,7 +43,7 @@ test() -> {userinfo,nil}, HostportBefore}, nil}, - {'query',nil}}} = URI_Before, + {query,nil}}} = URI_Before, %% ... some funky code ommitted, not relevant ... @@ -55,7 +55,7 @@ test() -> {userinfo,nil}, HostportAfter}, nil}, - {'query',nil}}} = URI_Before, + {query,nil}}} = URI_Before, %% NOTE: I intended to write URI_After instead of URI_Before %% but the accident revealed that when you add the line below, %% it causes internal error in v3_codegen on compilation diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index f647a4030d..aaa2005e73 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -32,7 +32,7 @@ binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1, other_output/1, kernel_listing/1, encrypted_abstr/1, strict_record/1, utf8_atoms/1, utf8_functions/1, extra_chunks/1, - cover/1, env/1, core_pp/1, + cover/1, env/1, core_pp/1, tuple_calls/1, core_roundtrip/1, asm/1, optimized_guards/1, sys_pre_attributes/1, dialyzer/1, warnings/1, pre_load_check/1, env_compiler_options/1, @@ -49,7 +49,7 @@ all() -> test_lib:recompile(?MODULE), [app_test, appup_test, file_1, forms_2, module_mismatch, big_file, outdir, binary, makedep, cond_and_ifdef, listings, listings_big, - other_output, kernel_listing, encrypted_abstr, + other_output, kernel_listing, encrypted_abstr, tuple_calls, strict_record, utf8_atoms, utf8_functions, extra_chunks, cover, env, core_pp, core_roundtrip, asm, optimized_guards, sys_pre_attributes, dialyzer, warnings, pre_load_check, @@ -781,6 +781,37 @@ extra_chunks(Config) when is_list(Config) -> {ok,{extra_chunks,[{"ExCh",<<"Contents">>}]}} = beam_lib:chunks(ExtraChunksBinary, ["ExCh"]). +tuple_calls(Config) when is_list(Config) -> + Anno = erl_anno:new(1), + Forms = [{attribute,Anno,export,[{size,1},{store,1}]}, + {function,Anno,size,1, + [{clause,Anno,[{var,[],mod}],[], + [{call,[],{remote,[],{var,[],mod},{atom,[],size}},[]}]}]}, + {function,Anno,store,1, + [{clause,Anno,[{var,[],mod}],[], + [{call,[],{remote,[],{var,[],mod},{atom,[],store}},[{atom,[],key},{atom,[],value}]}]}]}], + + TupleCallsFalse = [{attribute,Anno,module,tuple_calls_false}|Forms], + {ok,_,TupleCallsFalseBinary} = compile:forms(TupleCallsFalse, [binary]), + code:load_binary(tuple_calls_false, "compile_SUITE.erl", TupleCallsFalseBinary), + {'EXIT',{badarg,_}} = (catch tuple_calls_false:store(dict())), + {'EXIT',{badarg,_}} = (catch tuple_calls_false:size(dict())), + {'EXIT',{badarg,_}} = (catch tuple_calls_false:size(empty_tuple())), + + TupleCallsTrue = [{attribute,Anno,module,tuple_calls_true}|Forms], + {ok,_,TupleCallsTrueBinary} = compile:forms(TupleCallsTrue, [binary,tuple_calls]), + code:load_binary(tuple_calls_true, "compile_SUITE.erl", TupleCallsTrueBinary), + Dict = tuple_calls_true:store(dict()), + 1 = tuple_calls_true:size(Dict), + {'EXIT',{badarg,_}} = (catch tuple_calls_true:size(empty_tuple())), + + ok. + +dict() -> + dict:new(). +empty_tuple() -> + {}. + env(Config) when is_list(Config) -> {Simple,Target} = get_files(Config, simple, env), {ok,Cwd} = file:get_cwd(), diff --git a/lib/compiler/test/core_SUITE.erl b/lib/compiler/test/core_SUITE.erl index f8839da42f..0e07e8dd2e 100644 --- a/lib/compiler/test/core_SUITE.erl +++ b/lib/compiler/test/core_SUITE.erl @@ -28,7 +28,8 @@ map_core_test/1,eval_case/1,bad_boolean_guard/1, bs_shadowed_size_var/1, cover_v3_kernel_1/1,cover_v3_kernel_2/1,cover_v3_kernel_3/1, - cover_v3_kernel_4/1,cover_v3_kernel_5/1]). + cover_v3_kernel_4/1,cover_v3_kernel_5/1, + non_variable_apply/1]). -include_lib("common_test/include/ct.hrl"). @@ -56,7 +57,8 @@ groups() -> map_core_test,eval_case,bad_boolean_guard, bs_shadowed_size_var, cover_v3_kernel_1,cover_v3_kernel_2,cover_v3_kernel_3, - cover_v3_kernel_4,cover_v3_kernel_5 + cover_v3_kernel_4,cover_v3_kernel_5, + non_variable_apply ]}]. @@ -90,7 +92,7 @@ end_per_group(_GroupName, Config) -> ?comp(cover_v3_kernel_3). ?comp(cover_v3_kernel_4). ?comp(cover_v3_kernel_5). - +?comp(non_variable_apply). try_it(Mod, Conf) -> Src = filename:join(proplists:get_value(data_dir, Conf), diff --git a/lib/compiler/test/core_SUITE_data/non_variable_apply.core b/lib/compiler/test/core_SUITE_data/non_variable_apply.core new file mode 100644 index 0000000000..d9322cc455 --- /dev/null +++ b/lib/compiler/test/core_SUITE_data/non_variable_apply.core @@ -0,0 +1,80 @@ +module 'non_variable_apply' ['module_info'/0, + 'module_info'/1, + 'non_variable_apply'/0] + attributes [] + +'non_variable_apply'/0 = + %% Line 4 + fun () -> + case <> of + <> when 'true' -> + let <OkFun> = + fun (_@c0) -> + %% Line 5 + case _@c0 of + <'ok'> when 'true' -> + 'ok' + ( <_@c1> when 'true' -> + ( primop 'match_fail' + ({'function_clause',_@c1}) + -| [{'function_name',{'-non_variable_apply/0-fun-0-',1}}] ) + -| ['compiler_generated'] ) + end + in let <F> = + fun (_@c5,_@c4) -> + %% Line 6 + case <_@c5,_@c4> of + <F,X> when 'true' -> + apply apply 'id'/1 (F) (X) + ( <_@c7,_@c6> when 'true' -> + ( primop 'match_fail' + ({'function_clause',_@c7,_@c6}) + -| [{'function_name',{'-non_variable_apply/0-fun-1-',2}}] ) + -| ['compiler_generated'] ) + end + in %% Line 9 + apply F + (OkFun, 'ok') + ( <> when 'true' -> + ( primop 'match_fail' + ({'function_clause'}) + -| [{'function_name',{'non_variable_apply',0}}] ) + -| ['compiler_generated'] ) + end +'id'/1 = + %% Line 11 + fun (_@c0) -> + case _@c0 of + <I> when 'true' -> + I + ( <_@c1> when 'true' -> + ( primop 'match_fail' + ({'function_clause',_@c1}) + -| [{'function_name',{'id',1}}] ) + -| ['compiler_generated'] ) + end +'module_info'/0 = + fun () -> + case <> of + <> when 'true' -> + call 'erlang':'get_module_info' + ('non_variable_apply') + ( <> when 'true' -> + ( primop 'match_fail' + ({'function_clause'}) + -| [{'function_name',{'module_info',0}}] ) + -| ['compiler_generated'] ) + end +'module_info'/1 = + fun (_@c0) -> + case _@c0 of + <X> when 'true' -> + call 'erlang':'get_module_info' + ('non_variable_apply', X) + ( <_@c1> when 'true' -> + ( primop 'match_fail' + ({'function_clause',_@c1}) + -| [{'function_name',{'module_info',1}}] ) + -| ['compiler_generated'] ) + end +end diff --git a/lib/compiler/test/core_alias_SUITE.erl b/lib/compiler/test/core_alias_SUITE.erl new file mode 100644 index 0000000000..f3f15ef0f8 --- /dev/null +++ b/lib/compiler/test/core_alias_SUITE.erl @@ -0,0 +1,195 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(core_alias_SUITE). + +-export([all/0, suite/0, groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2, + tuples/1, cons/1]). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + test_lib:recompile(?MODULE), + [{group,p}]. + +groups() -> + [{p,[parallel], + [tuples, cons]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +id(X) -> X. + +tuples(Config) when is_list(Config) -> + Tuple = {ok,id(value)}, + + true = erts_debug:same(Tuple, simple_tuple(Tuple)), + true = erts_debug:same(Tuple, simple_tuple_in_map(#{hello => Tuple})), + true = erts_debug:same(Tuple, simple_tuple_case_repeated(Tuple, Tuple)), + true = erts_debug:same(Tuple, simple_tuple_fun_repeated(Tuple, Tuple)), + true = erts_debug:same(Tuple, simple_tuple_twice_head(Tuple, Tuple)), + + {Tuple1, Tuple2} = simple_tuple_twice_body(Tuple), + true = erts_debug:same(Tuple, Tuple1), + true = erts_debug:same(Tuple, Tuple2), + + Nested = {nested,Tuple}, + true = erts_debug:same(Tuple, nested_tuple_part(Nested)), + true = erts_debug:same(Nested, nested_tuple_whole(Nested)), + true = erts_debug:same(Nested, nested_tuple_with_alias(Nested)), + + true = erts_debug:same(Tuple, tuple_rebinding_after(Tuple)), + + Tuple = unaliased_tuple_rebinding_before(Tuple), + false = erts_debug:same(Tuple, unaliased_tuple_rebinding_before(Tuple)), + Nested = unaliased_literal_tuple_head(Nested), + false = erts_debug:same(Nested, unaliased_literal_tuple_head(Nested)), + Nested = unaliased_literal_tuple_body(Nested), + false = erts_debug:same(Nested, unaliased_literal_tuple_body(Nested)), + Nested = unaliased_different_var_tuple(Nested, Tuple), + false = erts_debug:same(Nested, unaliased_different_var_tuple(Nested, Tuple)). + +simple_tuple({ok,X}) -> + {ok,X}. +simple_tuple_twice_head({ok,X}, {ok,X}) -> + {ok,X}. +simple_tuple_twice_body({ok,X}) -> + {{ok,X},{ok,X}}. +simple_tuple_in_map(#{hello := {ok,X}}) -> + {ok,X}. +simple_tuple_fun_repeated({ok,X}, Y) -> + io:format("~p~n", [X]), + (fun({ok,X}) -> {ok,X} end)(Y). +simple_tuple_case_repeated({ok,X}, Y) -> + io:format("~p~n", [X]), + case Y of {ok,X} -> {ok,X} end. + +nested_tuple_part({nested,{ok,X}}) -> + {ok,X}. +nested_tuple_whole({nested,{ok,X}}) -> + {nested,{ok,X}}. +nested_tuple_with_alias({nested,{ok,_}=Y}) -> + {nested,Y}. + +tuple_rebinding_after(Y) -> + (fun(X) -> {ok,X} end)(Y), + case Y of {ok,X} -> {ok,X} end. +unaliased_tuple_rebinding_before({ok,X}) -> + io:format("~p~n", [X]), + (fun(X) -> {ok,X} end)(value). +unaliased_literal_tuple_head({nested,{ok,value}=X}) -> + io:format("~p~n", [X]), + {nested,{ok,value}}. +unaliased_literal_tuple_body({nested,{ok,value}=X}) -> + Res = {nested,Y={ok,value}}, + io:format("~p~n", [[X,Y]]), + Res. +unaliased_different_var_tuple({nested,{ok,value}=X}, Y) -> + io:format("~p~n", [X]), + {nested,Y}. + +cons(Config) when is_list(Config) -> + Cons = [ok|id(value)], + + true = erts_debug:same(Cons, simple_cons(Cons)), + true = erts_debug:same(Cons, simple_cons_in_map(#{hello => Cons})), + true = erts_debug:same(Cons, simple_cons_case_repeated(Cons, Cons)), + true = erts_debug:same(Cons, simple_cons_fun_repeated(Cons, Cons)), + true = erts_debug:same(Cons, simple_cons_twice_head(Cons, Cons)), + + {Cons1,Cons2} = simple_cons_twice_body(Cons), + true = erts_debug:same(Cons, Cons1), + true = erts_debug:same(Cons, Cons2), + + Nested = [nested,Cons], + true = erts_debug:same(Cons, nested_cons_part(Nested)), + true = erts_debug:same(Nested, nested_cons_whole(Nested)), + true = erts_debug:same(Nested, nested_cons_with_alias(Nested)), + true = erts_debug:same(Cons, cons_rebinding_after(Cons)), + + Unstripped = id([a,b]), + Stripped = cons_with_binary([<<>>|Unstripped]), + true = erts_debug:same(Unstripped, Stripped), + + Cons = unaliased_cons_rebinding_before(Cons), + false = erts_debug:same(Cons, unaliased_cons_rebinding_before(Cons)), + Nested = unaliased_literal_cons_head(Nested), + false = erts_debug:same(Nested, unaliased_literal_cons_head(Nested)), + Nested = unaliased_literal_cons_body(Nested), + false = erts_debug:same(Nested, unaliased_literal_cons_body(Nested)), + Nested = unaliased_different_var_cons(Nested, Cons), + false = erts_debug:same(Nested, unaliased_different_var_cons(Nested, Cons)). + +simple_cons([ok|X]) -> + [ok|X]. +simple_cons_twice_head([ok|X], [ok|X]) -> + [ok|X]. +simple_cons_twice_body([ok|X]) -> + {[ok|X],[ok|X]}. +simple_cons_in_map(#{hello := [ok|X]}) -> + [ok|X]. +simple_cons_fun_repeated([ok|X], Y) -> + io:format("~p~n", [X]), + (fun([ok|X]) -> [ok|X] end)(Y). +simple_cons_case_repeated([ok|X], Y) -> + io:format("~p~n", [X]), + case Y of [ok|X] -> [ok|X] end. + +nested_cons_part([nested,[ok|X]]) -> + [ok|X]. +nested_cons_whole([nested,[ok|X]]) -> + [nested,[ok|X]]. +nested_cons_with_alias([nested,[ok|_]=Y]) -> + [nested,Y]. + +cons_with_binary([<<>>,X|Y]) -> + cons_with_binary([X|Y]); +cons_with_binary(A) -> + A. + +cons_rebinding_after(Y) -> + (fun(X) -> [ok|X] end)(Y), + case Y of [ok|X] -> [ok|X] end. +unaliased_cons_rebinding_before([ok|X]) -> + io:format("~p~n", [X]), + (fun(X) -> [ok|X] end)(value). +unaliased_literal_cons_head([nested,[ok|value]=X]) -> + io:format("~p~n", [X]), + [nested,[ok|value]]. +unaliased_literal_cons_body([nested,[ok|value]=X]) -> + Res = [nested,Y=[ok|value]], + io:format("~p~n", [[X, Y]]), + Res. +unaliased_different_var_cons([nested,[ok|value]=X], Y) -> + io:format("~p~n", [X]), + [nested,Y]. diff --git a/lib/compiler/test/core_fold_SUITE.erl b/lib/compiler/test/core_fold_SUITE.erl index 0097e28d4d..262967d03d 100644 --- a/lib/compiler/test/core_fold_SUITE.erl +++ b/lib/compiler/test/core_fold_SUITE.erl @@ -26,7 +26,8 @@ unused_multiple_values_error/1,unused_multiple_values/1, multiple_aliases/1,redundant_boolean_clauses/1, mixed_matching_clauses/1,unnecessary_building/1, - no_no_file/1,configuration/1,supplies/1]). + no_no_file/1,configuration/1,supplies/1, + redundant_stack_frame/1]). -export([foo/0,foo/1,foo/2,foo/3]). @@ -45,7 +46,8 @@ groups() -> unused_multiple_values_error,unused_multiple_values, multiple_aliases,redundant_boolean_clauses, mixed_matching_clauses,unnecessary_building, - no_no_file,configuration,supplies]}]. + no_no_file,configuration,supplies, + redundant_stack_frame]}]. init_per_suite(Config) -> @@ -527,4 +529,26 @@ supplies(_Config) -> do_supplies(#{1 := Value}) when byte_size(Value), byte_size(kg) -> working. +redundant_stack_frame(_Config) -> + {1,2} = do_redundant_stack_frame(#{x=>1,y=>2}), + {'EXIT',{{badkey,_,x},_}} = (catch do_redundant_stack_frame(#{y=>2})), + {'EXIT',{{badkey,_,y},_}} = (catch do_redundant_stack_frame(#{x=>1})), + ok. + +do_redundant_stack_frame(Map) -> + %% There should not be a stack frame for this function. + X = case Map of + #{x := X0} -> + X0; + #{} -> + erlang:error({badkey, Map, x}) + end, + Y = case Map of + #{y := Y0} -> + Y0; + #{} -> + erlang:error({badkey, Map, y}) + end, + {X, Y}. + id(I) -> I. diff --git a/lib/compiler/test/match_SUITE.erl b/lib/compiler/test/match_SUITE.erl index 52b2da05f7..c31695be24 100644 --- a/lib/compiler/test/match_SUITE.erl +++ b/lib/compiler/test/match_SUITE.erl @@ -23,7 +23,7 @@ init_per_group/2,end_per_group/2, pmatch/1,mixed/1,aliases/1,non_matching_aliases/1, match_in_call/1,untuplify/1,shortcut_boolean/1,letify_guard/1, - selectify/1,underscore/1,match_map/1,map_vars_used/1, + selectify/1,deselectify/1,underscore/1,match_map/1,map_vars_used/1, coverage/1,grab_bag/1,literal_binary/1]). -include_lib("common_test/include/ct.hrl"). @@ -38,7 +38,7 @@ groups() -> [{p,[parallel], [pmatch,mixed,aliases,non_matching_aliases, match_in_call,untuplify, - shortcut_boolean,letify_guard,selectify, + shortcut_boolean,letify_guard,selectify,deselectify, underscore,match_map,map_vars_used,coverage, grab_bag,literal_binary]}]. @@ -466,6 +466,66 @@ sel_same_value2(V) when V =:= 42; V =:= 43 -> sel_same_value2(_) -> error. +%% Test deconstruction of select_val instructions in beam_peep into +%% regular tests with just one possible value left. Hitting proper cases +%% in beam_peep relies on unification of labels by beam_jump. + +deselectify(Config) when is_list(Config) -> + one_or_other = desel_tuple_arity({1}), + two = desel_tuple_arity({1,1}), + one_or_other = desel_tuple_arity({1,1,1}), + + one_or_other = dsel_integer(1), + two = dsel_integer(2), + one_or_other = dsel_integer(3), + + one_or_other = dsel_integer_typecheck(1), + two = dsel_integer_typecheck(2), + one_or_other = dsel_integer_typecheck(3), + + one_or_other = dsel_atom(one), + two = dsel_atom(two), + one_or_other = dsel_atom(three), + + one_or_other = dsel_atom_typecheck(one), + two = dsel_atom_typecheck(two), + one_or_other = dsel_atom_typecheck(three). + +desel_tuple_arity(Tuple) when is_tuple(Tuple) -> + case Tuple of + {_} -> one_or_other; + {_,_} -> two; + _ -> one_or_other + end. + +dsel_integer(Val) -> + case Val of + 1 -> one_or_other; + 2 -> two; + _ -> one_or_other + end. + +dsel_integer_typecheck(Val) when is_integer(Val) -> + case Val of + 1 -> one_or_other; + 2 -> two; + _ -> one_or_other + end. + +dsel_atom(Val) -> + case Val of + one -> one_or_other; + two -> two; + _ -> one_or_other + end. + +dsel_atom_typecheck(Val) when is_atom(Val) -> + case Val of + one -> one_or_other; + two -> two; + _ -> one_or_other + end. + underscore(Config) when is_list(Config) -> case Config of [] -> diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index 4bd884d86b..ea4aaf40a9 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -161,11 +161,12 @@ md5_1(Beam) -> %% Cover some code that handles internal errors. silly_coverage(Config) when is_list(Config) -> - %% sys_core_fold, sys_core_bsm, sys_core_setel, v3_kernel + %% sys_core_fold, sys_core_alias, sys_core_bsm, sys_core_setel, v3_kernel BadCoreErlang = {c_module,[], name,[],[], [{{c_var,[],{foo,2}},seriously_bad_body}]}, expect_error(fun() -> sys_core_fold:module(BadCoreErlang, []) end), + expect_error(fun() -> sys_core_alias:module(BadCoreErlang, []) end), expect_error(fun() -> sys_core_bsm:module(BadCoreErlang, []) end), expect_error(fun() -> sys_core_dsetel:module(BadCoreErlang, []) end), expect_error(fun() -> v3_kernel:module(BadCoreErlang, []) end), diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl index a591d6cc93..42dbf7d5f0 100644 --- a/lib/compiler/test/trycatch_SUITE.erl +++ b/lib/compiler/test/trycatch_SUITE.erl @@ -324,11 +324,11 @@ eclectic(Conf) when is_list(Conf) -> {{error,{exit,V},{'EXIT',V}},V} = eclectic_1({foo,{error,{exit,V}}}, error, {value,V}), {{value,{value,V},V}, - {'EXIT',{badarith,[{?MODULE,my_add,2,_}|_]}}} = + {'EXIT',{badarith,[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]}}} = eclectic_1({foo,{value,{value,V}}}, undefined, {'add',{0,a}}), {{'EXIT',V},V} = eclectic_1({catch_foo,{exit,V}}, undefined, {throw,V}), - {{error,{'div',{1,0}},{'EXIT',{badarith,[{?MODULE,my_div,2,_}|_]}}}, + {{error,{'div',{1,0}},{'EXIT',{badarith,[{erlang,'div',[1,0],_},{?MODULE,my_div,2,_}|_]}}}, {'EXIT',V}} = eclectic_1({foo,{error,{'div',{1,0}}}}, error, {exit,V}), {{{error,V},{'EXIT',{V,[{?MODULE,foo,1,_}|_]}}}, @@ -345,7 +345,7 @@ eclectic(Conf) when is_list(Conf) -> eclectic_2({error,{value,V}}, throw, {error,V}), {{caught,{'EXIT',{badarg,[{erlang,abs,[V],_}|_]}}},V} = eclectic_2({value,{'abs',V}}, undefined, {value,V}), - {{caught,{'EXIT',{badarith,[{?MODULE,my_add,2,_}|_]}}},V} = + {{caught,{'EXIT',{badarith,[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]}}},V} = eclectic_2({exit,{'add',{0,a}}}, exit, {value,V}), {{caught,{'EXIT',V}},undefined} = eclectic_2({value,{error,V}}, undefined, {exit,V}), diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index 463c264a5f..27ee5a3fb7 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.1 +COMPILER_VSN = 7.1.1 diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 688ec339aa..1d9c1e0f88 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -110,6 +110,10 @@ #endif +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) +# define HAS_EVP_PKEY_CTX +#endif + #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) #include <openssl/modes.h> @@ -433,13 +437,11 @@ static ERL_NIF_TERM strong_rand_bytes_nif(ErlNifEnv* env, int argc, const ERL_NI static ERL_NIF_TERM strong_rand_range_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM dss_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rsa_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM do_exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rc4_set_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rc4_encrypt_with_state(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM dss_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM pkey_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_public_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_private_crypt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rsa_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -452,8 +454,6 @@ static ERL_NIF_TERM srp_user_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_ static ERL_NIF_TERM srp_host_secret_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM ecdsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM ecdsa_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM ecdh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rand_seed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -506,13 +506,11 @@ static ErlNifFunc nif_funcs[] = { {"strong_rand_range_nif", 1, strong_rand_range_nif}, {"rand_uniform_nif", 2, rand_uniform_nif}, {"mod_exp_nif", 4, mod_exp_nif}, - {"dss_verify_nif", 4, dss_verify_nif}, - {"rsa_verify_nif", 4, rsa_verify_nif}, {"do_exor", 2, do_exor}, {"rc4_set_key", 1, rc4_set_key}, {"rc4_encrypt_with_state", 2, rc4_encrypt_with_state}, - {"rsa_sign_nif", 3, rsa_sign_nif}, - {"dss_sign_nif", 3, dss_sign_nif}, + {"pkey_sign_nif", 5, pkey_sign_nif}, + {"pkey_verify_nif", 6, pkey_verify_nif}, {"rsa_public_crypt", 4, rsa_public_crypt}, {"rsa_private_crypt", 4, rsa_private_crypt}, {"rsa_generate_key_nif", 2, rsa_generate_key_nif}, @@ -525,8 +523,6 @@ static ErlNifFunc nif_funcs[] = { {"srp_host_secret_nif", 5, srp_host_secret_nif}, {"ec_key_generate", 2, ec_key_generate}, - {"ecdsa_sign_nif", 4, ecdsa_sign_nif}, - {"ecdsa_verify_nif", 5, ecdsa_verify_nif}, {"ecdh_compute_key_nif", 3, ecdh_compute_key_nif}, {"rand_seed_nif", 1, rand_seed_nif}, @@ -589,6 +585,23 @@ static ERL_NIF_TERM atom_des_ecb; static ERL_NIF_TERM atom_blowfish_ecb; #endif +static ERL_NIF_TERM atom_rsa; +static ERL_NIF_TERM atom_dss; +static ERL_NIF_TERM atom_ecdsa; +static ERL_NIF_TERM atom_rsa_mgf1_md; +static ERL_NIF_TERM atom_rsa_padding; +static ERL_NIF_TERM atom_rsa_pkcs1_pss_padding; +static ERL_NIF_TERM atom_rsa_x931_padding; +static ERL_NIF_TERM atom_rsa_pss_saltlen; +static ERL_NIF_TERM atom_sha224; +static ERL_NIF_TERM atom_sha256; +static ERL_NIF_TERM atom_sha384; +static ERL_NIF_TERM atom_sha512; +static ERL_NIF_TERM atom_md5; +static ERL_NIF_TERM atom_ripemd160; + + + static ErlNifResourceType* hmac_context_rtype; struct hmac_context { @@ -916,6 +929,20 @@ static int initialize(ErlNifEnv* env, ERL_NIF_TERM load_info) #else atom_not_supported = enif_make_atom(env,"not_supported"); #endif + atom_rsa = enif_make_atom(env,"rsa"); + atom_dss = enif_make_atom(env,"dss"); + atom_ecdsa = enif_make_atom(env,"ecdsa"); + atom_rsa_mgf1_md = enif_make_atom(env,"rsa_mgf1_md"); + atom_rsa_padding = enif_make_atom(env,"rsa_padding"); + atom_rsa_pkcs1_pss_padding = enif_make_atom(env,"rsa_pkcs1_pss_padding"); + atom_rsa_x931_padding = enif_make_atom(env,"rsa_x931_padding"); + atom_rsa_pss_saltlen = enif_make_atom(env,"rsa_pss_saltlen"); + atom_sha224 = enif_make_atom(env,"sha224"); + atom_sha256 = enif_make_atom(env,"sha256"); + atom_sha384 = enif_make_atom(env,"sha384"); + atom_sha512 = enif_make_atom(env,"sha512"); + atom_md5 = enif_make_atom(env,"md5"); + atom_ripemd160 = enif_make_atom(env,"ripemd160"); init_digest_types(env); init_cipher_types(env); @@ -1010,6 +1037,8 @@ static int algo_pubkey_cnt, algo_pubkey_fips_cnt; static ERL_NIF_TERM algo_pubkey[7]; /* increase when extending the list */ static int algo_cipher_cnt, algo_cipher_fips_cnt; static ERL_NIF_TERM algo_cipher[24]; /* increase when extending the list */ +static int algo_mac_cnt, algo_mac_fips_cnt; +static ERL_NIF_TERM algo_mac[2]; /* increase when extending the list */ static void init_algorithms_types(ErlNifEnv* env) { @@ -1093,9 +1122,19 @@ static void init_algorithms_types(ErlNifEnv* env) algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"chacha20_poly1305"); #endif + // Validated algorithms first + algo_mac_cnt = 0; + algo_mac[algo_mac_cnt++] = enif_make_atom(env,"hmac"); +#ifdef HAVE_CMAC + algo_mac[algo_mac_cnt++] = enif_make_atom(env,"cmac"); +#endif + // Non-validated algorithms follow + algo_mac_fips_cnt = algo_mac_cnt; + ASSERT(algo_hash_cnt <= sizeof(algo_hash)/sizeof(ERL_NIF_TERM)); ASSERT(algo_pubkey_cnt <= sizeof(algo_pubkey)/sizeof(ERL_NIF_TERM)); ASSERT(algo_cipher_cnt <= sizeof(algo_cipher)/sizeof(ERL_NIF_TERM)); + ASSERT(algo_mac_cnt <= sizeof(algo_mac)/sizeof(ERL_NIF_TERM)); } static ERL_NIF_TERM algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -1105,15 +1144,19 @@ static ERL_NIF_TERM algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv int hash_cnt = fips_mode ? algo_hash_fips_cnt : algo_hash_cnt; int pubkey_cnt = fips_mode ? algo_pubkey_fips_cnt : algo_pubkey_cnt; int cipher_cnt = fips_mode ? algo_cipher_fips_cnt : algo_cipher_cnt; + int mac_cnt = fips_mode ? algo_mac_fips_cnt : algo_mac_cnt; #else int hash_cnt = algo_hash_cnt; int pubkey_cnt = algo_pubkey_cnt; int cipher_cnt = algo_cipher_cnt; + int mac_cnt = algo_mac_cnt; #endif - return enif_make_tuple3(env, + return enif_make_tuple4(env, enif_make_list_from_array(env, algo_hash, hash_cnt), enif_make_list_from_array(env, algo_pubkey, pubkey_cnt), - enif_make_list_from_array(env, algo_cipher, cipher_cnt)); + enif_make_list_from_array(env, algo_cipher, cipher_cnt), + enif_make_list_from_array(env, algo_mac, mac_cnt) + ); } static ERL_NIF_TERM info_lib(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -2448,44 +2491,6 @@ static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg return ret; } -static ERL_NIF_TERM dss_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (sha, Digest, Signature,Key=[P, Q, G, Y]) */ - ErlNifBinary digest_bin, sign_bin; - BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL, *dsa_y = NULL; - ERL_NIF_TERM head, tail; - DSA *dsa; - int i; - - if (argv[0] != atom_sha - || !enif_inspect_binary(env, argv[1], &digest_bin) - || digest_bin.size != SHA_DIGEST_LENGTH - || !enif_inspect_binary(env, argv[2], &sign_bin) - || !enif_get_list_cell(env, argv[3], &head, &tail) - || !get_bn_from_bin(env, head, &dsa_p) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_q) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_g) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &dsa_y) - || !enif_is_empty_list(env,tail)) { - - if (dsa_p) BN_free(dsa_p); - if (dsa_q) BN_free(dsa_q); - if (dsa_g) BN_free(dsa_g); - if (dsa_y) BN_free(dsa_y); - return enif_make_badarg(env); - } - - dsa = DSA_new(); - DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g); - DSA_set0_key(dsa, dsa_y, NULL); - i = DSA_verify(0, digest_bin.data, SHA_DIGEST_LENGTH, - sign_bin.data, sign_bin.size, dsa); - DSA_free(dsa); - return(i > 0) ? atom_true : atom_false; -} - static void init_digest_types(ErlNifEnv* env) { struct digest_type_t* p = digest_types; @@ -2532,73 +2537,6 @@ static struct cipher_type_t* get_cipher_type(ERL_NIF_TERM type, size_t key_len) return NULL; } -static ERL_NIF_TERM rsa_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Digest, Signature, Key=[E,N]) */ - ErlNifBinary digest_bin, sign_bin; - ERL_NIF_TERM head, tail, ret; - int i; - RSA *rsa; -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - EVP_PKEY *pkey; - EVP_PKEY_CTX *ctx; -#endif - const EVP_MD *md; - const ERL_NIF_TERM type = argv[0]; - struct digest_type_t *digp = NULL; - BIGNUM *rsa_e; - BIGNUM *rsa_n; - - digp = get_digest_type(type); - if (!digp) { - return enif_make_badarg(env); - } - md = digp->md.p; - if (!md) { - return atom_notsup; - } - - rsa = RSA_new(); - - if (!enif_inspect_binary(env, argv[1], &digest_bin) - || digest_bin.size != EVP_MD_size(md) - || !enif_inspect_binary(env, argv[2], &sign_bin) - || !enif_get_list_cell(env, argv[3], &head, &tail) - || !get_bn_from_bin(env, head, &rsa_e) - || !enif_get_list_cell(env, tail, &head, &tail) - || !get_bn_from_bin(env, head, &rsa_n) - || !enif_is_empty_list(env, tail)) { - - ret = enif_make_badarg(env); - goto done; - } - - (void) RSA_set0_key(rsa, rsa_n, rsa_e, NULL); - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - pkey = EVP_PKEY_new(); - EVP_PKEY_set1_RSA(pkey, rsa); - - ctx = EVP_PKEY_CTX_new(pkey, NULL); - EVP_PKEY_verify_init(ctx); - EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING); - EVP_PKEY_CTX_set_signature_md(ctx, md); - - i = EVP_PKEY_verify(ctx, sign_bin.data, sign_bin.size, - digest_bin.data, digest_bin.size); - EVP_PKEY_CTX_free(ctx); - EVP_PKEY_free(pkey); -#else - i = RSA_verify(md->type, digest_bin.data, EVP_MD_size(md), - sign_bin.data, sign_bin.size, rsa); -#endif - - ret = (i==1 ? atom_true : atom_false); - -done: - RSA_free(rsa); - return ret; -} - static ERL_NIF_TERM do_exor(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {/* (Data1, Data2) */ ErlNifBinary d1, d2; @@ -2702,100 +2640,33 @@ static int get_rsa_private_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) return 1; } -static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Digest, Key=[E,N,D]|[E,N,D,P1,P2,E1,E2,C]) */ - ErlNifBinary digest_bin, ret_bin; -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - EVP_PKEY *pkey; - EVP_PKEY_CTX *ctx; - size_t rsa_s_len; -#else - unsigned rsa_s_len, len; -#endif - RSA *rsa; - int i; - struct digest_type_t *digp; - const EVP_MD *md; - - digp = get_digest_type(argv[0]); - if (!digp) { - return enif_make_badarg(env); - } - md = digp->md.p; - if (!md) { - return atom_notsup; - } - if (!enif_inspect_binary(env,argv[1],&digest_bin) - || digest_bin.size != EVP_MD_size(md)) { - return enif_make_badarg(env); - } +static int get_rsa_public_key(ErlNifEnv* env, ERL_NIF_TERM key, RSA *rsa) +{ + /* key=[E,N] */ + ERL_NIF_TERM head, tail; + BIGNUM *e, *n; - rsa = RSA_new(); - if (!get_rsa_private_key(env, argv[2], rsa)) { - RSA_free(rsa); - return enif_make_badarg(env); + if (!enif_get_list_cell(env, key, &head, &tail) + || !get_bn_from_bin(env, head, &e) + || !enif_get_list_cell(env, tail, &head, &tail) + || !get_bn_from_bin(env, head, &n) + || !enif_is_empty_list(env, tail)) { + return 0; } - -#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) - pkey = EVP_PKEY_new(); - EVP_PKEY_set1_RSA(pkey, rsa); - rsa_s_len=(size_t)EVP_PKEY_size(pkey); - enif_alloc_binary(rsa_s_len, &ret_bin); - - ctx = EVP_PKEY_CTX_new(pkey, NULL); - EVP_PKEY_sign_init(ctx); - EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING); - EVP_PKEY_CTX_set_signature_md(ctx, md); - - i = EVP_PKEY_sign(ctx, ret_bin.data, &rsa_s_len, - digest_bin.data, digest_bin.size); - ASSERT(i<=0 || rsa_s_len <= ret_bin.size); - EVP_PKEY_CTX_free(ctx); - EVP_PKEY_free(pkey); -#else - enif_alloc_binary(RSA_size(rsa), &ret_bin); - len = EVP_MD_size(md); - - ERL_VALGRIND_ASSERT_MEM_DEFINED(digest_bin.data, len); - i = RSA_sign(md->type, digest_bin.data, len, - ret_bin.data, &rsa_s_len, rsa); -#endif - - RSA_free(rsa); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(ret_bin.data, rsa_s_len); - if (rsa_s_len != ret_bin.size) { - enif_realloc_binary(&ret_bin, rsa_s_len); - ERL_VALGRIND_ASSERT_MEM_DEFINED(ret_bin.data, rsa_s_len); - } - return enif_make_binary(env,&ret_bin); - } - else { - enif_release_binary(&ret_bin); - return atom_error; - } + (void) RSA_set0_key(rsa, n, e, NULL); + return 1; } - -static ERL_NIF_TERM dss_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (sha, Digest, Key=[P,Q,G,PrivKey]) */ - ErlNifBinary digest_bin, ret_bin; +static int get_dss_private_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) +{ + /* key=[P,Q,G,KEY] */ ERL_NIF_TERM head, tail; - unsigned int dsa_s_len; - DSA* dsa; BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL; BIGNUM *dummy_pub_key, *priv_key = NULL; - int i; - - if (argv[0] != atom_sha - || !enif_inspect_binary(env, argv[1], &digest_bin) - || digest_bin.size != SHA_DIGEST_LENGTH) { - return enif_make_badarg(env); - } - if (!enif_get_list_cell(env, argv[2], &head, &tail) + if (!enif_get_list_cell(env, key, &head, &tail) || !get_bn_from_bin(env, head, &dsa_p) || !enif_get_list_cell(env, tail, &head, &tail) || !get_bn_from_bin(env, head, &dsa_q) @@ -2808,7 +2679,7 @@ static ERL_NIF_TERM dss_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar if (dsa_q) BN_free(dsa_q); if (dsa_g) BN_free(dsa_g); if (priv_key) BN_free(priv_key); - return enif_make_badarg(env); + return 0; } /* Note: DSA_set0_key() does not allow setting only the @@ -2818,27 +2689,39 @@ static ERL_NIF_TERM dss_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar */ dummy_pub_key = BN_dup(priv_key); - dsa = DSA_new(); DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g); DSA_set0_key(dsa, dummy_pub_key, priv_key); - enif_alloc_binary(DSA_size(dsa), &ret_bin); - i = DSA_sign(NID_sha1, digest_bin.data, SHA_DIGEST_LENGTH, - ret_bin.data, &dsa_s_len, dsa); - DSA_free(dsa); - - if (i) { - if (dsa_s_len != ret_bin.size) { - enif_realloc_binary(&ret_bin, dsa_s_len); - } - return enif_make_binary(env, &ret_bin); - } - else { - enif_release_binary(&ret_bin); - return atom_error; - } + return 1; } +static int get_dss_public_key(ErlNifEnv* env, ERL_NIF_TERM key, DSA *dsa) +{ + /* key=[P, Q, G, Y] */ + ERL_NIF_TERM head, tail; + BIGNUM *dsa_p = NULL, *dsa_q = NULL, *dsa_g = NULL, *dsa_y = NULL; + + if (!enif_get_list_cell(env, key, &head, &tail) + || !get_bn_from_bin(env, head, &dsa_p) + || !enif_get_list_cell(env, tail, &head, &tail) + || !get_bn_from_bin(env, head, &dsa_q) + || !enif_get_list_cell(env, tail, &head, &tail) + || !get_bn_from_bin(env, head, &dsa_g) + || !enif_get_list_cell(env, tail, &head, &tail) + || !get_bn_from_bin(env, head, &dsa_y) + || !enif_is_empty_list(env,tail)) { + if (dsa_p) BN_free(dsa_p); + if (dsa_q) BN_free(dsa_q); + if (dsa_g) BN_free(dsa_g); + if (dsa_y) BN_free(dsa_y); + return 0; + } + + DSA_set0_pqg(dsa, dsa_p, dsa_q, dsa_g); + DSA_set0_key(dsa, dsa_y, NULL); + return 1; +} + static int rsa_pad(ERL_NIF_TERM term, int* padding) { if (term == atom_rsa_pkcs1_padding) { @@ -3788,99 +3671,6 @@ badarg: #endif } -static ERL_NIF_TERM ecdsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Digest, Curve, Key) */ -#if defined(HAVE_EC) - ErlNifBinary digest_bin, ret_bin; - unsigned int dsa_s_len; - EC_KEY* key = NULL; - int i, len; - struct digest_type_t *digp; - const EVP_MD *md; - - digp = get_digest_type(argv[0]); - if (!digp) { - return enif_make_badarg(env); - } - md = digp->md.p; - if (!md) { - return atom_notsup; - } - len = EVP_MD_size(md); - - if (!enif_inspect_binary(env,argv[1],&digest_bin) - || digest_bin.size != len - || !get_ec_key(env, argv[2], argv[3], atom_undefined, &key)) - goto badarg; - - enif_alloc_binary(ECDSA_size(key), &ret_bin); - - i = ECDSA_sign(EVP_MD_type(md), digest_bin.data, len, - ret_bin.data, &dsa_s_len, key); - - EC_KEY_free(key); - if (i) { - if (dsa_s_len != ret_bin.size) { - enif_realloc_binary(&ret_bin, dsa_s_len); - } - return enif_make_binary(env, &ret_bin); - } - else { - enif_release_binary(&ret_bin); - return atom_error; - } - -badarg: - if (key) - EC_KEY_free(key); - return make_badarg_maybe(env); -#else - return atom_notsup; -#endif -} - -static ERL_NIF_TERM ecdsa_verify_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{/* (Type, Digest, Signature, Curve, Key) */ -#if defined(HAVE_EC) - ErlNifBinary digest_bin, sign_bin; - int i, len; - EC_KEY* key = NULL; - const ERL_NIF_TERM type = argv[0]; - struct digest_type_t *digp = NULL; - const EVP_MD *md; - - digp = get_digest_type(type); - if (!digp) { - return enif_make_badarg(env); - } - md = digp->md.p; - if (!md) { - return atom_notsup; - } - len = EVP_MD_size(md); - - if (!enif_inspect_binary(env, argv[1], &digest_bin) - || digest_bin.size != len - || !enif_inspect_binary(env, argv[2], &sign_bin) - || !get_ec_key(env, argv[3], atom_undefined, argv[4], &key)) - goto badarg; - - i = ECDSA_verify(EVP_MD_type(md), digest_bin.data, len, - sign_bin.data, sign_bin.size, key); - - EC_KEY_free(key); - - return (i==1 ? atom_true : atom_false); - -badarg: - if (key) - EC_KEY_free(key); - return make_badarg_maybe(env); -#else - return atom_notsup; -#endif -} - /* (_OthersPublicKey, _MyPrivateKey) (_OthersPublicKey, _MyEC_Point) @@ -3939,6 +3729,548 @@ out_err: #endif } +/*================================================================*/ +#define PKEY_BADARG -1 +#define PKEY_NOTSUP 0 +#define PKEY_OK 1 + +typedef struct PKeyCryptOptions { + const EVP_MD *rsa_mgf1_md; + ErlNifBinary rsa_oaep_label; + const EVP_MD *rsa_oaep_md; + int rsa_padding; + const EVP_MD *signature_md; +} PKeyCryptOptions; + +typedef struct PKeySignOptions { + const EVP_MD *rsa_mgf1_md; + int rsa_padding; + int rsa_pss_saltlen; +} PKeySignOptions; + +static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM type, + const EVP_MD **md) +{ + struct digest_type_t *digp = NULL; + *md = NULL; + + if (type == atom_none && algorithm == atom_rsa) return PKEY_OK; + + digp = get_digest_type(type); + if (!digp) return PKEY_BADARG; + if (!digp->md.p) return PKEY_NOTSUP; + + *md = digp->md.p; + return PKEY_OK; +} + + +static int get_pkey_sign_digest(ErlNifEnv *env, ERL_NIF_TERM algorithm, + ERL_NIF_TERM type, ERL_NIF_TERM data, + unsigned char *md_value, const EVP_MD **mdp, + unsigned char **tbsp, size_t *tbslenp) +{ + int i; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + ErlNifBinary tbs_bin; + EVP_MD_CTX *mdctx; + const EVP_MD *md = *mdp; + unsigned char *tbs = *tbsp; + size_t tbslen = *tbslenp; + unsigned int tbsleni; + + if ((i = get_pkey_digest_type(env, algorithm, type, &md)) != PKEY_OK) { + return i; + } + if (enif_get_tuple(env, data, &tpl_arity, &tpl_terms)) { + if (tpl_arity != 2 || tpl_terms[0] != atom_digest + || !enif_inspect_binary(env, tpl_terms[1], &tbs_bin) + || (md != NULL && tbs_bin.size != EVP_MD_size(md))) { + return PKEY_BADARG; + } + /* We have a digest (= hashed text) in tbs_bin */ + tbs = tbs_bin.data; + tbslen = tbs_bin.size; + } else if (md == NULL) { + if (!enif_inspect_binary(env, data, &tbs_bin)) { + return PKEY_BADARG; + } + /* md == NULL, that is no hashing because DigestType argument was atom_none */ + tbs = tbs_bin.data; + tbslen = tbs_bin.size; + } else { + if (!enif_inspect_binary(env, data, &tbs_bin)) { + return PKEY_BADARG; + } + /* We have the cleartext in tbs_bin and the hash algo info in md */ + tbs = md_value; + mdctx = EVP_MD_CTX_create(); + if (!mdctx) { + return PKEY_BADARG; + } + /* Looks well, now hash the plain text into a digest according to md */ + if (EVP_DigestInit_ex(mdctx, md, NULL) <= 0) { + EVP_MD_CTX_destroy(mdctx); + return PKEY_BADARG; + } + if (EVP_DigestUpdate(mdctx, tbs_bin.data, tbs_bin.size) <= 0) { + EVP_MD_CTX_destroy(mdctx); + return PKEY_BADARG; + } + if (EVP_DigestFinal_ex(mdctx, tbs, &tbsleni) <= 0) { + EVP_MD_CTX_destroy(mdctx); + return PKEY_BADARG; + } + tbslen = (size_t)(tbsleni); + EVP_MD_CTX_destroy(mdctx); + } + + *mdp = md; + *tbsp = tbs; + *tbslenp = tbslen; + + return PKEY_OK; +} + + +static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, + const EVP_MD *md, PKeySignOptions *opt) +{ + ERL_NIF_TERM head, tail; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + const EVP_MD *opt_md; + int i; + + if (!enif_is_list(env, options)) { + return PKEY_BADARG; + } + + /* defaults */ + if (algorithm == atom_rsa) { + opt->rsa_mgf1_md = NULL; + opt->rsa_padding = RSA_PKCS1_PADDING; + opt->rsa_pss_saltlen = -2; + } + + if (enif_is_empty_list(env, options)) { + return PKEY_OK; + } + + if (algorithm == atom_rsa) { + tail = options; + while (enif_get_list_cell(env, tail, &head, &tail)) { + if (enif_get_tuple(env, head, &tpl_arity, &tpl_terms) && tpl_arity == 2) { + if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->rsa_mgf1_md = opt_md; + } else if (tpl_terms[0] == atom_rsa_padding) { + if (tpl_terms[1] == atom_rsa_pkcs1_padding) { + opt->rsa_padding = RSA_PKCS1_PADDING; + } else if (tpl_terms[1] == atom_rsa_pkcs1_pss_padding) { +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,0) + opt->rsa_padding = RSA_PKCS1_PSS_PADDING; + if (opt->rsa_mgf1_md == NULL) { + opt->rsa_mgf1_md = md; + } +#else + return PKEY_NOTSUP; +#endif + } else if (tpl_terms[1] == atom_rsa_x931_padding) { + opt->rsa_padding = RSA_X931_PADDING; + } else if (tpl_terms[1] == atom_rsa_no_padding) { + opt->rsa_padding = RSA_NO_PADDING; + } else { + return PKEY_BADARG; + } + } else if (tpl_terms[0] == atom_rsa_pss_saltlen) { + if (!enif_get_int(env, tpl_terms[1], &(opt->rsa_pss_saltlen)) + || opt->rsa_pss_saltlen < -2) { + return PKEY_BADARG; + } + } else { + return PKEY_BADARG; + } + } else { + return PKEY_BADARG; + } + } + } else { + return PKEY_BADARG; + } + + return PKEY_OK; +} + +static int get_pkey_sign_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, EVP_PKEY **pkey) +{ + if (algorithm == atom_rsa) { + RSA *rsa = RSA_new(); + + if (!get_rsa_private_key(env, key, rsa)) { + RSA_free(rsa); + return PKEY_BADARG; + } + + *pkey = EVP_PKEY_new(); + if (!EVP_PKEY_assign_RSA(*pkey, rsa)) { + EVP_PKEY_free(*pkey); + RSA_free(rsa); + return PKEY_BADARG; + } + } else if (algorithm == atom_ecdsa) { +#if defined(HAVE_EC) + EC_KEY *ec = NULL; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + + if (enif_get_tuple(env, key, &tpl_arity, &tpl_terms) && tpl_arity == 2 + && enif_is_tuple(env, tpl_terms[0]) && enif_is_binary(env, tpl_terms[1]) + && get_ec_key(env, tpl_terms[0], tpl_terms[1], atom_undefined, &ec)) { + + *pkey = EVP_PKEY_new(); + if (!EVP_PKEY_assign_EC_KEY(*pkey, ec)) { + EVP_PKEY_free(*pkey); + EC_KEY_free(ec); + return PKEY_BADARG; + } + } else { + return PKEY_BADARG; + } +#else + return PKEY_NOTSUP; +#endif + } else if (algorithm == atom_dss) { + DSA *dsa = DSA_new(); + + if (!get_dss_private_key(env, key, dsa)) { + DSA_free(dsa); + return PKEY_BADARG; + } + + *pkey = EVP_PKEY_new(); + if (!EVP_PKEY_assign_DSA(*pkey, dsa)) { + EVP_PKEY_free(*pkey); + DSA_free(dsa); + return PKEY_BADARG; + } + } else { + return PKEY_BADARG; + } + + return PKEY_OK; +} + +static ERL_NIF_TERM pkey_sign_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{/* (Algorithm, Type, Data|{digest,Digest}, Key, Options) */ + int i; + const EVP_MD *md = NULL; + unsigned char md_value[EVP_MAX_MD_SIZE]; + EVP_PKEY *pkey; +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx; + size_t siglen; +#else + unsigned len, siglen; +#endif + PKeySignOptions sig_opt; + ErlNifBinary sig_bin; /* signature */ + unsigned char *tbs; /* data to be signed */ + size_t tbslen; +/*char buf[1024]; +enif_get_atom(env,argv[0],buf,1024,ERL_NIF_LATIN1); printf("algo=%s ",buf); +enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf); +printf("\r\n"); +*/ + i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); + if (i != PKEY_OK) { + if (i == PKEY_NOTSUP) + return atom_notsup; + else + return enif_make_badarg(env); + } + + i = get_pkey_sign_options(env, argv[0], argv[4], md, &sig_opt); + if (i != PKEY_OK) { + if (i == PKEY_NOTSUP) + return atom_notsup; + else + return enif_make_badarg(env); + } + + if (get_pkey_sign_key(env, argv[0], argv[3], &pkey) != PKEY_OK) { + return enif_make_badarg(env); + } + +#ifdef HAS_EVP_PKEY_CTX +/* printf("EVP interface\r\n"); + */ + ctx = EVP_PKEY_CTX_new(pkey, NULL); + if (!ctx) goto badarg; + if (EVP_PKEY_sign_init(ctx) <= 0) goto badarg; + if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + + if (argv[0] == atom_rsa) { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; + if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { + if (sig_opt.rsa_mgf1_md != NULL) { +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,1) + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg; +#else + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return atom_notsup; +#endif + } + if (sig_opt.rsa_pss_saltlen > -2 + && EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0) + goto badarg; + } + } + + if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) <= 0) goto badarg; + enif_alloc_binary(siglen, &sig_bin); + + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + } + i = EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen); + + EVP_PKEY_CTX_free(ctx); +#else +/*printf("Old interface\r\n"); + */ + if (argv[0] == atom_rsa) { + RSA *rsa = EVP_PKEY_get1_RSA(pkey); + enif_alloc_binary(RSA_size(rsa), &sig_bin); + len = EVP_MD_size(md); + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + i = RSA_sign(md->type, tbs, len, sig_bin.data, &siglen, rsa); + RSA_free(rsa); + } else if (argv[0] == atom_dss) { + DSA *dsa = EVP_PKEY_get1_DSA(pkey); + enif_alloc_binary(DSA_size(dsa), &sig_bin); + len = EVP_MD_size(md); + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + i = DSA_sign(md->type, tbs, len, sig_bin.data, &siglen, dsa); + DSA_free(dsa); + } else if (argv[0] == atom_ecdsa) { +#if defined(HAVE_EC) + EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); + enif_alloc_binary(ECDSA_size(ec), &sig_bin); + len = EVP_MD_size(md); + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + i = ECDSA_sign(md->type, tbs, len, sig_bin.data, &siglen, ec); + EC_KEY_free(ec); +#else + EVP_PKEY_free(pkey); + return atom_notsup; +#endif + } else { + goto badarg; + } +#endif + + EVP_PKEY_free(pkey); + if (i == 1) { + ERL_VALGRIND_MAKE_MEM_DEFINED(sig_bin.data, siglen); + if (siglen != sig_bin.size) { + enif_realloc_binary(&sig_bin, siglen); + ERL_VALGRIND_ASSERT_MEM_DEFINED(sig_bin.data, siglen); + } + return enif_make_binary(env, &sig_bin); + } else { + enif_release_binary(&sig_bin); + return atom_error; + } + + badarg: +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX_free(ctx); +#endif + EVP_PKEY_free(pkey); + return enif_make_badarg(env); +} + + +static int get_pkey_verify_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, + EVP_PKEY **pkey) +{ + if (algorithm == atom_rsa) { + RSA *rsa = RSA_new(); + + if (!get_rsa_public_key(env, key, rsa)) { + RSA_free(rsa); + return PKEY_BADARG; + } + + *pkey = EVP_PKEY_new(); + if (!EVP_PKEY_assign_RSA(*pkey, rsa)) { + EVP_PKEY_free(*pkey); + RSA_free(rsa); + return PKEY_BADARG; + } + } else if (algorithm == atom_ecdsa) { +#if defined(HAVE_EC) + EC_KEY *ec = NULL; + const ERL_NIF_TERM *tpl_terms; + int tpl_arity; + + if (enif_get_tuple(env, key, &tpl_arity, &tpl_terms) && tpl_arity == 2 + && enif_is_tuple(env, tpl_terms[0]) && enif_is_binary(env, tpl_terms[1]) + && get_ec_key(env, tpl_terms[0], atom_undefined, tpl_terms[1], &ec)) { + + *pkey = EVP_PKEY_new(); + if (!EVP_PKEY_assign_EC_KEY(*pkey, ec)) { + EVP_PKEY_free(*pkey); + EC_KEY_free(ec); + return PKEY_BADARG; + } + } else { + return PKEY_BADARG; + } +#else + return PKEY_NOTSUP; +#endif + } else if (algorithm == atom_dss) { + DSA *dsa = DSA_new(); + + if (!get_dss_public_key(env, key, dsa)) { + DSA_free(dsa); + return PKEY_BADARG; + } + + *pkey = EVP_PKEY_new(); + if (!EVP_PKEY_assign_DSA(*pkey, dsa)) { + EVP_PKEY_free(*pkey); + DSA_free(dsa); + return PKEY_BADARG; + } + } else { + return PKEY_BADARG; + } + + return PKEY_OK; +} + +static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{/* (Algorithm, Type, Data|{digest,Digest}, Signature, Key, Options) */ + int i; + const EVP_MD *md = NULL; + unsigned char md_value[EVP_MAX_MD_SIZE]; + EVP_PKEY *pkey; +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX *ctx; +#else +#endif + PKeySignOptions sig_opt; + ErlNifBinary sig_bin; /* signature */ + unsigned char *tbs; /* data to be signed */ + size_t tbslen; + + if (!enif_inspect_binary(env, argv[3], &sig_bin)) { + return enif_make_badarg(env); + } + + i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); + if (i != PKEY_OK) { + if (i == PKEY_NOTSUP) + return atom_notsup; + else + return enif_make_badarg(env); + } + + i = get_pkey_sign_options(env, argv[0], argv[5], md, &sig_opt); + if (i != PKEY_OK) { + if (i == PKEY_NOTSUP) + return atom_notsup; + else + return enif_make_badarg(env); + } + + if (get_pkey_verify_key(env, argv[0], argv[4], &pkey) != PKEY_OK) { + return enif_make_badarg(env); + } + +#ifdef HAS_EVP_PKEY_CTX +/* printf("EVP interface\r\n"); + */ + ctx = EVP_PKEY_CTX_new(pkey, NULL); + if (!ctx) goto badarg; + if (EVP_PKEY_verify_init(ctx) <= 0) goto badarg; + if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + + if (argv[0] == atom_rsa) { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; + if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { + if (sig_opt.rsa_mgf1_md != NULL) { +#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,1) + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg; +#else + EVP_PKEY_CTX_free(ctx); + EVP_PKEY_free(pkey); + return atom_notsup; +#endif + } + if (sig_opt.rsa_pss_saltlen > -2 + && EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0) + goto badarg; + } + } + + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); + } + i = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); + + EVP_PKEY_CTX_free(ctx); +#else +/*printf("Old interface\r\n"); +*/ + if (argv[0] == atom_rsa) { + RSA *rsa = EVP_PKEY_get1_RSA(pkey); + i = RSA_verify(md->type, tbs, tbslen, sig_bin.data, sig_bin.size, rsa); + RSA_free(rsa); + } else if (argv[0] == atom_dss) { + DSA *dsa = EVP_PKEY_get1_DSA(pkey); + i = DSA_verify(0, tbs, tbslen, sig_bin.data, sig_bin.size, dsa); + DSA_free(dsa); + } else if (argv[0] == atom_ecdsa) { +#if defined(HAVE_EC) + EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); + i = ECDSA_verify(EVP_MD_type(md), tbs, tbslen, sig_bin.data, sig_bin.size, ec); + EC_KEY_free(ec); +#else + EVP_PKEY_free(pkey); + return atom_notsup; +#endif + } else { + goto badarg; + } +#endif + + EVP_PKEY_free(pkey); + if (i == 1) { + return atom_true; + } else { + return atom_false; + } + + badarg: +#ifdef HAS_EVP_PKEY_CTX + EVP_PKEY_CTX_free(ctx); +#endif + EVP_PKEY_free(pkey); + return enif_make_badarg(env); +} + + +/*================================================================*/ + static ERL_NIF_TERM rand_seed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary seed_bin; diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml index 89ef529c5d..5b2c46a004 100644 --- a/lib/crypto/doc/src/crypto.xml +++ b/lib/crypto/doc/src/crypto.xml @@ -159,12 +159,24 @@ <code>digest_type() = md5 | sha | sha224 | sha256 | sha384 | sha512</code> + <code>rsa_digest_type() = md5 | ripemd160 | sha | sha224 | sha256 | sha384 | sha512</code> + + <code>dss_digest_type() = sha | sha224 | sha256 | sha384 | sha512</code> <p>Note that the actual supported + dss_digest_type depends on the underlying crypto library. In OpenSSL version >= 1.0.1 the listed digest are supported, while in 1.0.0 only sha, sha224 and sha256 are supported. In version 0.9.8 only sha is supported.</p> + + <code>ecdsa_digest_type() = sha | sha224 | sha256 | sha384 | sha512</code> + + <code>sign_options() = [{rsa_pad, rsa_sign_padding()} | {rsa_pss_saltlen, integer()}]</code> + + <code>rsa_sign_padding() = rsa_pkcs1_padding | rsa_pkcs1_pss_padding</code> + <code> hash_algorithms() = md5 | ripemd160 | sha | sha224 | sha256 | sha384 | sha512 </code> <p>md4 is also supported for hash_init/1 and hash/2. Note that both md4 and md5 are recommended only for compatibility with existing applications. </p> <code> cipher_algorithms() = aes_cbc | aes_cfb8 | aes_cfb128 | aes_ctr | aes_gcm | aes_ige256 | blowfish_cbc | blowfish_cfb64 | chacha20_poly1305 | des_cbc | des_cfb | des3_cbc | des3_cfb | des_ede3 | rc2_cbc | rc4 </code> + <code> mac_algorithms() = hmac | cmac</code> <code> public_key_algorithms() = rsa |dss | ecdsa | dh | ecdh | ec_gf2m</code> <p>Note that ec_gf2m is not strictly a public key algorithm, but a restriction on what curves are supported with ecdsa and ecdh. @@ -681,6 +693,7 @@ <func> <name>sign(Algorithm, DigestType, Msg, Key) -> binary()</name> + <name>sign(Algorithm, DigestType, Msg, Key, Options) -> binary()</name> <fsummary> Create digital signature.</fsummary> <type> <v>Algorithm = rsa | dss | ecdsa </v> @@ -688,8 +701,9 @@ <d>The msg is either the binary "cleartext" data to be signed or it is the hashed value of "cleartext" i.e. the digest (plaintext).</d> - <v>DigestType = digest_type()</v> + <v>DigestType = rsa_digest_type() | dss_digest_type() | ecdsa_digest_type()</v> <v>Key = rsa_private() | dss_private() | [ecdh_private(),ecdh_params()]</v> + <v>Options = sign_options()</v> </type> <desc> <p>Creates a digital signature.</p> @@ -835,7 +849,8 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> <type> <v> AlgorithmList = [{hashs, [hash_algorithms()]}, {ciphers, [cipher_algorithms()]}, - {public_keys, [public_key_algorithms()]} + {public_keys, [public_key_algorithms()]}, + {macs, [mac_algorithms()]}] </v> </type> <desc> @@ -869,15 +884,17 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre> <func> <name>verify(Algorithm, DigestType, Msg, Signature, Key) -> boolean()</name> + <name>verify(Algorithm, DigestType, Msg, Signature, Key, Options) -> boolean()</name> <fsummary>Verifies a digital signature.</fsummary> <type> <v> Algorithm = rsa | dss | ecdsa </v> <v>Msg = binary() | {digest,binary()}</v> <d>The msg is either the binary "cleartext" data or it is the hashed value of "cleartext" i.e. the digest (plaintext).</d> - <v>DigestType = digest_type()</v> + <v>DigestType = rsa_digest_type() | dss_digest_type() | ecdsa_digest_type()</v> <v>Signature = binary()</v> <v>Key = rsa_public() | dss_public() | [ecdh_public(),ecdh_params()]</v> + <v>Options = sign_options()</v> </type> <desc> <p>Verifies a digital signature</p> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index d111525214..1df05462c9 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -25,7 +25,7 @@ -export([start/0, stop/0, info_lib/0, info_fips/0, supports/0, enable_fips_mode/1, version/0, bytes_to_integer/1]). -export([hash/2, hash_init/1, hash_update/2, hash_final/1]). --export([sign/4, verify/5]). +-export([sign/4, sign/5, verify/5, verify/6]). -export([generate_key/2, generate_key/3, compute_key/4]). -export([hmac/3, hmac/4, hmac_init/2, hmac_update/2, hmac_final/1, hmac_final_n/2]). -export([cmac/3, cmac/4]). @@ -45,6 +45,10 @@ -export([ec_curve/1, ec_curves/0]). -export([rand_seed/1]). +%% Private. For tests. +-export([packed_openssl_version/4]). + + -deprecated({rand_uniform, 2, next_major_release}). %% This should correspond to the similar macro in crypto.c @@ -87,11 +91,12 @@ stop() -> application:stop(crypto). supports()-> - {Hashs, PubKeys, Ciphers} = algorithms(), + {Hashs, PubKeys, Ciphers, Macs} = algorithms(), [{hashs, Hashs}, {ciphers, Ciphers}, - {public_keys, PubKeys} + {public_keys, PubKeys}, + {macs, Macs} ]. info_lib() -> ?nif_stub. @@ -388,36 +393,31 @@ mod_pow(Base, Exponent, Prime) -> <<0>> -> error; R -> R end. -verify(dss, none, Data, Signature, Key) when is_binary(Data) -> - verify(dss, sha, {digest, Data}, Signature, Key); -verify(Alg, Type, Data, Signature, Key) when is_binary(Data) -> - verify(Alg, Type, {digest, hash(Type, Data)}, Signature, Key); -verify(dss, Type, {digest, Digest}, Signature, Key) -> - dss_verify_nif(Type, Digest, Signature, map_ensure_int_as_bin(Key)); -verify(rsa, Type, {digest, Digest}, Signature, Key) -> - notsup_to_error( - rsa_verify_nif(Type, Digest, Signature, map_ensure_int_as_bin(Key))); -verify(ecdsa, Type, {digest, Digest}, Signature, [Key, Curve]) -> - notsup_to_error( - ecdsa_verify_nif(Type, Digest, Signature, nif_curve_params(Curve), ensure_int_as_bin(Key))). -sign(dss, none, Data, Key) when is_binary(Data) -> - sign(dss, sha, {digest, Data}, Key); -sign(Alg, Type, Data, Key) when is_binary(Data) -> - sign(Alg, Type, {digest, hash(Type, Data)}, Key); -sign(rsa, Type, {digest, Digest}, Key) -> - case rsa_sign_nif(Type, Digest, map_ensure_int_as_bin(Key)) of - error -> erlang:error(badkey, [rsa, Type, {digest, Digest}, Key]); - Sign -> Sign - end; -sign(dss, Type, {digest, Digest}, Key) -> - case dss_sign_nif(Type, Digest, map_ensure_int_as_bin(Key)) of - error -> erlang:error(badkey, [dss, Type, {digest, Digest}, Key]); - Sign -> Sign - end; -sign(ecdsa, Type, {digest, Digest}, [Key, Curve]) -> - case ecdsa_sign_nif(Type, Digest, nif_curve_params(Curve), ensure_int_as_bin(Key)) of - error -> erlang:error(badkey, [ecdsa, Type, {digest, Digest}, [Key, Curve]]); - Sign -> Sign + +verify(Algorithm, Type, Data, Signature, Key) -> + verify(Algorithm, Type, Data, Signature, Key, []). + +%% Backwards compatible +verify(Algorithm = dss, none, Digest, Signature, Key, Options) -> + verify(Algorithm, sha, {digest, Digest}, Signature, Key, Options); +verify(Algorithm, Type, Data, Signature, Key, Options) -> + case pkey_verify_nif(Algorithm, Type, Data, Signature, format_pkey(Algorithm, Key), Options) of + notsup -> erlang:error(notsup); + Boolean -> Boolean + end. + + +sign(Algorithm, Type, Data, Key) -> + sign(Algorithm, Type, Data, Key, []). + +%% Backwards compatible +sign(Algorithm = dss, none, Digest, Key, Options) -> + sign(Algorithm, sha, {digest, Digest}, Key, Options); +sign(Algorithm, Type, Data, Key, Options) -> + case pkey_sign_nif(Algorithm, Type, Data, format_pkey(Algorithm, Key), Options) of + error -> erlang:error(badkey, [Algorithm, Type, Data, Key, Options]); + notsup -> erlang:error(notsup); + Signature -> Signature end. -spec public_encrypt(rsa, binary(), [binary()], rsa_padding()) -> @@ -838,13 +838,9 @@ srp_value_B_nif(_Multiplier, _Verifier, _Generator, _Exponent, _Prime) -> ?nif_s %% Digital signatures -------------------------------------------------------------------- -rsa_sign_nif(_Type,_Digest,_Key) -> ?nif_stub. -dss_sign_nif(_Type,_Digest,_Key) -> ?nif_stub. -ecdsa_sign_nif(_Type, _Digest, _Curve, _Key) -> ?nif_stub. -dss_verify_nif(_Type, _Digest, _Signature, _Key) -> ?nif_stub. -rsa_verify_nif(_Type, _Digest, _Signature, _Key) -> ?nif_stub. -ecdsa_verify_nif(_Type, _Digest, _Signature, _Curve, _Key) -> ?nif_stub. +pkey_sign_nif(_Algorithm, _Type, _Digest, _Key, _Options) -> ?nif_stub. +pkey_verify_nif(_Algorithm, _Type, _Data, _Signature, _Key, _Options) -> ?nif_stub. %% Public Keys -------------------------------------------------------------------- %% RSA Rivest-Shamir-Adleman functions @@ -961,6 +957,15 @@ ensure_int_as_bin(Int) when is_integer(Int) -> ensure_int_as_bin(Bin) -> Bin. +format_pkey(rsa, Key) -> + map_ensure_int_as_bin(Key); +format_pkey(ecdsa, [Key, Curve]) -> + {nif_curve_params(Curve), ensure_int_as_bin(Key)}; +format_pkey(dss, Key) -> + map_ensure_int_as_bin(Key); +format_pkey(_, Key) -> + Key. + %%-------------------------------------------------------------------- %% -type rsa_padding() :: 'rsa_pkcs1_padding' | 'rsa_pkcs1_oaep_padding' | 'rsa_no_padding'. @@ -1003,3 +1008,14 @@ erlint(<<MPIntSize:32/integer,MPIntValue/binary>>) -> %% mod_exp_nif(_Base,_Exp,_Mod,_bin_hdr) -> ?nif_stub. + +%%%---------------------------------------------------------------- +%% 9470495 == V(0,9,8,zh). +%% 268435615 == V(1,0,0,i). +%% 268439663 == V(1,0,1,f). + +packed_openssl_version(MAJ, MIN, FIX, P0) -> + %% crypto.c + P1 = atom_to_list(P0), + P = lists:sum([C-$a||C<-P1]), + ((((((((MAJ bsl 8) bor MIN) bsl 8 ) bor FIX) bsl 8) bor (P+1)) bsl 4) bor 16#f). diff --git a/lib/crypto/test/blowfish_SUITE.erl b/lib/crypto/test/blowfish_SUITE.erl index c2d0d2621b..c9033ac4f8 100644 --- a/lib/crypto/test/blowfish_SUITE.erl +++ b/lib/crypto/test/blowfish_SUITE.erl @@ -47,6 +47,11 @@ init_per_suite(Config) -> case catch crypto:start() of ok -> + catch ct:comment("~s",[element(3,hd(crypto:info_lib()))]), + catch ct:log("crypto:info_lib() -> ~p~n" + "crypto:supports() -> ~p~n" + "crypto:version() -> ~p~n" + ,[crypto:info_lib(), crypto:supports(), crypto:version()]), Config; _Else -> {skip,"Could not start crypto!"} diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 164f43dcb0..88f13d766c 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -177,6 +177,12 @@ init_per_suite(Config) -> try crypto:start() of ok -> + catch ct:comment("~s",[element(3,hd(crypto:info_lib()))]), + catch ct:log("crypto:info_lib() -> ~p~n" + "crypto:supports() -> ~p~n" + "crypto:version() -> ~p~n" + ,[crypto:info_lib(), crypto:supports(), crypto:version()]), + try crypto:strong_rand_bytes(1) of _ -> Config @@ -745,10 +751,44 @@ do_sign_verify({Type, Hash, Public, Private, Msg}) -> Signature = crypto:sign(Type, Hash, Msg, Private), case crypto:verify(Type, Hash, Msg, Signature, Public) of true -> + ct:log("OK crypto:sign(~p, ~p, ..., ..., ...)", [Type,Hash]), negative_verify(Type, Hash, Msg, <<10,20>>, Public); false -> + ct:log("ERROR crypto:sign(~p, ~p, ..., ..., ...)", [Type,Hash]), ct:fail({{crypto, verify, [Type, Hash, Msg, Signature, Public]}}) - end. + end; +do_sign_verify({Type, Hash, Public, Private, Msg, Options}) -> + LibVer = + case crypto:info_lib() of + [{<<"OpenSSL">>,Ver,<<"OpenSSL",_/binary>>}] -> Ver; + _ -> infinity + end, + Pad = proplists:get_value(rsa_padding, Options), + NotSupLow = lists:member(Pad, [rsa_pkcs1_pss_padding]), + try + crypto:sign(Type, Hash, Msg, Private, Options) + of + Signature -> + case crypto:verify(Type, Hash, Msg, Signature, Public, Options) of + true -> + ct:log("OK crypto:sign(~p, ~p, ..., ..., ..., ~p)", [Type,Hash,Options]), + negative_verify(Type, Hash, Msg, <<10,20>>, Public, Options); + false -> + ct:log("ERROR crypto:sign(~p, ~p, ..., ..., ..., ~p)", [Type,Hash,Options]), + ct:fail({{crypto, verify, [Type, Hash, Msg, Signature, Public, Options]}}) + end + catch + error:notsup when NotSupLow == true, + is_integer(LibVer), + LibVer < 16#10001000 -> + %% Thoose opts where introduced in 1.0.1 + ct:log("notsup but OK in old cryptolib crypto:sign(~p, ~p, ..., ..., ..., ~p)", + [Type,Hash,Options]), + true; + C:E -> + ct:log("~p:~p crypto:sign(~p, ~p, ..., ..., ..., ~p)", [C,E,Type,Hash,Options]), + ct:fail({{crypto, sign_verify, [LibVer, Type, Hash, Msg, Public, Options]}}) + end. negative_verify(Type, Hash, Msg, Signature, Public) -> case crypto:verify(Type, Hash, Msg, Signature, Public) of @@ -758,6 +798,14 @@ negative_verify(Type, Hash, Msg, Signature, Public) -> ok end. +negative_verify(Type, Hash, Msg, Signature, Public, Options) -> + case crypto:verify(Type, Hash, Msg, Signature, Public, Options) of + true -> + ct:fail({{crypto, verify, [Type, Hash, Msg, Signature, Public, Options]}, should_fail}); + false -> + ok + end. + do_public_encrypt({Type, Public, Private, Msg, Padding}) -> PublicEcn = (catch crypto:public_encrypt(Type, Msg, Public, Padding)), case crypto:private_decrypt(Type, PublicEcn, Private, Padding) of @@ -1172,13 +1220,29 @@ group_config(dss = Type, Config) -> Msg = dss_plain(), Public = dss_params() ++ [dss_public()], Private = dss_params() ++ [dss_private()], - SignVerify = [{Type, sha, Public, Private, Msg}], + SupportedHashs = proplists:get_value(hashs, crypto:supports(), []), + DssHashs = + case crypto:info_lib() of + [{<<"OpenSSL">>,LibVer,_}] when is_integer(LibVer), LibVer > 16#10001000 -> + [sha, sha224, sha256, sha384, sha512]; + [{<<"OpenSSL">>,LibVer,_}] when is_integer(LibVer), LibVer > 16#10000000 -> + [sha, sha224, sha256]; + _Else -> + [sha] + end, + SignVerify = [{Type, Hash, Public, Private, Msg} + || Hash <- DssHashs, + lists:member(Hash, SupportedHashs)], [{sign_verify, SignVerify} | Config]; group_config(ecdsa = Type, Config) -> {Private, Public} = ec_key_named(), Msg = ec_msg(), - SignVerify = [{Type, sha, Public, Private, Msg}], + SupportedHashs = proplists:get_value(hashs, crypto:supports(), []), + DssHashs = [sha, sha224, sha256, sha384, sha512], + SignVerify = [{Type, Hash, Public, Private, Msg} + || Hash <- DssHashs, + lists:member(Hash, SupportedHashs)], [{sign_verify, SignVerify} | Config]; group_config(srp, Config) -> GenerateCompute = [srp3(), srp6(), srp6a(), srp6a_smaller_prime()], @@ -1262,18 +1326,38 @@ group_config(_, Config) -> Config. sign_verify_tests(Type, Msg, Public, Private, PublicS, PrivateS) -> - sign_verify_tests(Type, [md5, sha, sha224, sha256], Msg, Public, Private) ++ - sign_verify_tests(Type, [sha384, sha512], Msg, PublicS, PrivateS). - -sign_verify_tests(Type, Hashs, Msg, Public, Private) -> - lists:foldl(fun(Hash, Acc) -> - case is_supported(Hash) of - true -> - [{Type, Hash, Public, Private, Msg}|Acc]; - false -> - Acc - end - end, [], Hashs). + gen_sign_verify_tests(Type, [md5, ripemd160, sha, sha224, sha256], Msg, Public, Private, + [undefined, + [{rsa_padding, rsa_pkcs1_pss_padding}], + [{rsa_padding, rsa_pkcs1_pss_padding}, {rsa_pss_saltlen, 0}], + [{rsa_padding, rsa_x931_padding}] + ]) ++ + gen_sign_verify_tests(Type, [sha384, sha512], Msg, PublicS, PrivateS, + [undefined, + [{rsa_padding, rsa_pkcs1_pss_padding}], + [{rsa_padding, rsa_pkcs1_pss_padding}, {rsa_pss_saltlen, 0}], + [{rsa_padding, rsa_x931_padding}] + ]). + +gen_sign_verify_tests(Type, Hashs, Msg, Public, Private, Opts) -> + lists:foldr(fun(Hash, Acc0) -> + case is_supported(Hash) of + true -> + lists:foldr(fun + (undefined, Acc1) -> + [{Type, Hash, Public, Private, Msg} | Acc1]; + ([{rsa_padding, rsa_x931_padding} | _], Acc1) + when Hash =:= md5 + orelse Hash =:= ripemd160 + orelse Hash =:= sha224 -> + Acc1; + (Opt, Acc1) -> + [{Type, Hash, Public, Private, Msg, Opt} | Acc1] + end, Acc0, Opts); + false -> + Acc0 + end + end, [], Hashs). rfc_1321_msgs() -> [<<"">>, @@ -2294,7 +2378,7 @@ fmt_words(Words) -> log_rsp_size(Label, Term) -> S = erts_debug:size(Term), - ct:pal("~s: ~w test(s), Memory used: ~s", + ct:log("~s: ~w test(s), Memory used: ~s", [Label, length(Term), fmt_words(S)]). read_rsp(Config, Type, Files) -> diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 0d2cb6c4df..c26b7aab5e 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug where merging PLT:s could lose info. The + bug was introduced in Erlang/OTP 20.0. </p> + <p> + Own Id: OTP-14558 Aux Id: ERIERL-53 </p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 47994fc35b..0fd99bbc04 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -772,6 +772,7 @@ tab_is_disj(K1, T1, T2) -> end. merge_tables(T1, T2) -> + ets:safe_fixtable(T1, true), tab_merge(ets:first(T1), T1, T2). tab_merge('$end_of_table', T1, T2) -> diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 511a6d66bf..6e501f32b2 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -120,92 +120,10 @@ get_core_from_beam(File, Opts) -> {error, " Could not get Core Erlang code for: " ++ File ++ "\n"} end; _ -> - deprecated_get_core_from_beam(File, Opts) + {error, " Could not get Core Erlang code for: " ++ File ++ "\n" ++ + " Recompile with +debug_info or analyze starting from source code"} end. -deprecated_get_core_from_beam(File, Opts) -> - case get_abstract_code_from_beam(File) of - error -> - {error, " Could not get abstract code for: " ++ File ++ "\n" ++ - " Recompile with +debug_info or analyze starting from source code"}; - {ok, AbstrCode} -> - case get_compile_options_from_beam(File) of - error -> - {error, " Could not get compile options for: " ++ File ++ "\n" ++ - " Recompile or analyze starting from source code"}; - {ok, CompOpts} -> - case get_core_from_abstract_code(AbstrCode, Opts ++ CompOpts) of - error -> - {error, " Could not get core Erlang code for: " ++ File}; - {ok, _} = Core -> - Core - end - end - end. - -get_abstract_code_from_beam(File) -> - case beam_lib:chunks(File, [abstract_code]) of - {ok, {_, List}} -> - case lists:keyfind(abstract_code, 1, List) of - {abstract_code, {raw_abstract_v1, Abstr}} -> {ok, Abstr}; - _ -> error - end; - _ -> - %% No or unsuitable abstract code. - error - end. - -get_compile_options_from_beam(File) -> - case beam_lib:chunks(File, [compile_info]) of - {ok, {_, List}} -> - case lists:keyfind(compile_info, 1, List) of - {compile_info, CompInfo} -> compile_info_to_options(CompInfo); - _ -> error - end; - _ -> - %% No or unsuitable compile info. - error - end. - -compile_info_to_options(CompInfo) -> - case lists:keyfind(options, 1, CompInfo) of - {options, CompOpts} -> {ok, CompOpts}; - _ -> error - end. - -get_core_from_abstract_code(AbstrCode, Opts) -> - %% We do not want the parse_transforms around since we already - %% performed them. In some cases we end up in trouble when - %% performing them again. - AbstrCode1 = cleanup_parse_transforms(AbstrCode), - %% Remove parse_transforms (and other options) from compile options. - Opts2 = cleanup_compile_options(Opts), - try compile:noenv_forms(AbstrCode1, Opts2 ++ src_compiler_opts()) of - {ok, _, Core} -> {ok, Core}; - _What -> error - catch - error:_ -> error - end. - -cleanup_parse_transforms([{attribute, _, compile, {parse_transform, _}}|Left]) -> - cleanup_parse_transforms(Left); -cleanup_parse_transforms([Other|Left]) -> - [Other|cleanup_parse_transforms(Left)]; -cleanup_parse_transforms([]) -> - []. - -cleanup_compile_options(Opts) -> - lists:filter(fun keep_compile_option/1, Opts). - -%% Using abstract, not asm or core. -keep_compile_option(from_asm) -> false; -keep_compile_option(from_core) -> false; -%% The parse transform will already have been applied, may cause -%% problems if it is re-applied. -keep_compile_option({parse_transform, _}) -> false; -keep_compile_option(warnings_as_errors) -> false; -keep_compile_option(_) -> true. - %% ============================================================================ %% %% Typed Records diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 4a1a7c25a0..866a82ee3e 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 3.2 +DIALYZER_VSN = 3.2.1 diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 2cbe48ecce..6b84b22eb5 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -397,10 +397,10 @@ from the peer offers it.</p> Note that each tuple communicates one or more AVP values. It is an error to specify duplicate tuples.</p> -<marker id="evaluable"/> +<marker id="eval"/> </item> -<tag><c>evaluable() = {M,F,A} | fun() | [evaluable() | A]</c></tag> +<tag><c>eval() = {M,F,A} | fun() | [eval() | A]</c></tag> <item> <p> An expression that can be evaluated as a function in the following @@ -418,7 +418,7 @@ eval(F) -> </pre> <p> -Applying an <c>&evaluable;</c> +Applying an <c>&eval;</c> <c>E</c> to an argument list <c>A</c> is meant in the sense of <c>eval([E|A])</c>.</p> @@ -484,11 +484,11 @@ Matches only those peers whose Origin-Realm has the specified value, or all peers if the atom <c>any</c>.</p> </item> -<tag><c>{eval, &evaluable;}</c></tag> +<tag><c>{eval, &eval;}</c></tag> <item> <p> Matches only those peers for which the specified -<c>&evaluable;</c> returns +<c>&eval;</c> returns <c>true</c> when applied to the connection's <c>diameter_caps</c> record. Any other return value or exception is equivalent to <c>false</c>.</p> @@ -650,7 +650,7 @@ Result = ResultCode | {capabilities_cb, CB, ResultCode|discard} Caps = #diameter_caps{} Pkt = #diameter_packet{} ResultCode = integer() -CB = &evaluable; +CB = &eval; </pre> <p> @@ -798,15 +798,31 @@ be matched by corresponding &capability; configuration, of </item> <tag> -<marker id="incoming_maxlen"/><c>{incoming_maxlen, 0..16777215}</c></tag> +<marker id="decode_format"/> +<c>{decode_format, record | list | map | none}</c></tag> <item> <p> -Bound on the expected size of incoming Diameter messages. -Messages larger than the specified number of bytes are discarded.</p> +The format of decoded messages and grouped AVPs in the <c>msg</c> field +of diameter_packet records and <c>value</c> field of diameter_avp +records respectively. +If <c>record</c> then a record whose definition is generated from the +dictionary file in question. +If <c>list</c> or <c>map</c> then a <c>[Name | Avps]</c> pair where +<c>Avps</c> is a list of AVP name/values pairs or a map keyed on +AVP names respectively. +If <c>none</c> then the atom-value message name, or <c>undefined</c> +for a Grouped AVP. +See also &codec_message;.</p> <p> -Defaults to <c>16777215</c>, the maximum value of the 24-bit Message -Length field in a Diameter Header.</p> +Defaults to <c>record</c>.</p> + +<note> +<p> +AVPs are decoded into a list of diameter_avp records in <c>avps</c> +field of diameter_packet records independently of +<c>decode_format</c>.</p> +</note> </item> @@ -814,7 +830,7 @@ Length field in a Diameter Header.</p> | node | nodes | [node()] - | evaluable()}</c></tag> + | eval()}</c></tag> <item> <p> The degree to which the service allows multiple transport @@ -825,7 +841,7 @@ at capabilities exchange.</p> If <c>[node()]</c> then a connection is rejected if another already exists on any of the specified nodes. Types <c>false</c>, <c>node</c>, <c>nodes</c> and -&evaluable; are equivalent to +&eval; are equivalent to <c>[]</c>, <c>[node()]</c>, <c>[node()|nodes()]</c> and the evaluated value respectively, evaluation of each expression taking place whenever a new connection is to be established. @@ -840,7 +856,7 @@ by their own peer and watchdog state machines.</p> Defaults to <c>nodes</c>.</p> </item> -<tag><c>{sequence, {H,N} | &evaluable;}</c></tag> +<tag><c>{sequence, {H,N} | &eval;}</c></tag> <item> <p> A constant value <c>H</c> for the topmost <c>32-N</c> bits of @@ -875,7 +891,7 @@ outgoing requests.</p> </warning> </item> -<tag><c>{share_peers, boolean() | [node()] | evaluable()}</c></tag> +<tag><c>{share_peers, boolean() | [node()] | eval()}</c></tag> <item> <p> Nodes to which peer connections established on the local @@ -888,7 +904,7 @@ configured to use them: see <c>use_shared_peers</c> below.</p> If <c>false</c> then peers are not shared. If <c>[node()]</c> then peers are shared with the specified list of nodes. -If <c>evaluable()</c> then peers are shared with the nodes returned +If <c>eval()</c> then peers are shared with the nodes returned by the specified function, evaluated whenever a peer connection becomes available or a remote service requests information about local connections. @@ -914,59 +930,36 @@ of a single Diameter node across multiple Erlang nodes.</p> </note> </item> -<tag><c>{spawn_opt, [term()]}</c></tag> -<item> -<p> -Options list passed to &spawn_opt; when spawning a process for an -incoming Diameter request, unless the transport in question -specifies another value. -Options <c>monitor</c> and <c>link</c> are ignored.</p> - -<p> -Defaults to the empty list.</p> -</item> - <tag> -<marker id="strict_mbit"/><c>{strict_mbit, boolean()}</c></tag> +<marker id="strict_arities"/><c>{strict_arities, boolean() + | encode + | decode}</c></tag> <item> <p> -Whether or not to regard an AVP setting the M-bit as erroneous when -the command grammar in question does not explicitly allow the AVP. -If <c>true</c> then such AVPs are regarded as 5001 errors, -DIAMETER_AVP_UNSUPPORTED. -If <c>false</c> then the M-bit is ignored and policing -it becomes the receiver's responsibility.</p> +Whether or not to require that the number of AVPs in a message or +grouped AVP agree with those specified in the dictionary in question +when passing messages to &man_app; callbacks. +If <c>true</c> then mismatches in an outgoing messages cause message +encoding to fail, while mismatches in an incoming message are reported +as 5005/5009 errors in the errors field of the diameter_packet record +passed to &app_handle_request; or &app_handle_answer; callbacks. +If <c>false</c> then neither error is enforced/detected. +If <c>encode</c> or <c>decode</c> then errors are only +enforced/detected on outgoing or incoming messages respectively.</p> <p> Defaults to <c>true</c>.</p> -<warning> -<p> -RFC 6733 is unclear about the semantics of the M-bit. -One the one hand, the CCF specification in section 3.2 documents AVP -in a command grammar as meaning <em>any</em> arbitrary AVP; on the -other hand, 1.3.4 states that AVPs setting the M-bit cannot be added -to an existing command: the modified command must instead be -placed in a new Diameter application.</p> -<p> -The reason for the latter is presumably interoperability: -allowing arbitrary AVPs setting the M-bit in a command makes its -interpretation implementation-dependent, since there's no -guarantee that all implementations will understand the same set of -arbitrary AVPs in the context of a given command. -However, interpreting <c>AVP</c> in a command grammar as any -AVP, regardless of M-bit, renders 1.3.4 meaningless, since the receiver -can simply ignore any AVP it thinks isn't relevant, regardless of the -sender's intent.</p> +<note> <p> -Beware of confusing mandatory in the sense of the M-bit with mandatory -in the sense of the command grammar. -The former is a semantic requirement: that the receiver understand the -semantics of the AVP in the context in question. -The latter is a syntactic requirement: whether or not the AVP must -occur in the message in question.</p> -</warning> - +Disabling arity checks affects the form of messages at encode/decode. +In particular, decoded AVPs are represented as lists of values, +regardless of the AVP's arity (ie. expected number in the message/AVP +grammar in question), and values are expected to be supplied as lists +at encode. +This differs from the historic decode behaviour of representing AVPs +of arity 1 as bare values, not wrapped in a list.</p> +</note> </item> <tag> @@ -993,7 +986,27 @@ The default value is for backwards compatibility.</p> </item> -<tag><c>{use_shared_peers, boolean() | [node()] | evaluable()}</c></tag> +<tag> +<marker id="traffic_counters"/><c>{traffic_counters, boolean()}</c></tag> +<item> +<p> +Whether or not to count application-specific messages; those for which +&man_app; callbacks take place. +If false then only messages handled by diameter itself are counted: +CER/CEA, DWR/DWA, DPR/DPA.</p> + +<p> +Defaults to <c>true</c>.</p> + +<note> +<p> +Disabling counters is a performance improvement, but means that the +omitted counters are not returned by &service_info;.</p> +</note> + +</item> + +<tag><c>{use_shared_peers, boolean() | [node()] | eval()}</c></tag> <item> <p> Nodes from which communicated peers are made available in @@ -1003,7 +1016,7 @@ the remote candidates list of &app_pick_peer; callbacks.</p> If <c>false</c> then remote peers are not used. If <c>[node()]</c> then only peers from the specified list of nodes are used. -If <c>evaluable()</c> then only peers returned by the specified +If <c>eval()</c> then only peers returned by the specified function are used, evaluated whenever a remote service communicates information about an available peer connection. The value <c>true</c> is equivalent to <c>fun &nodes;</c>. @@ -1028,6 +1041,15 @@ each node from which requests are sent.</p> </warning> </item> +<tag><c>&transport_opt;</c></tag> +<item> +<p> +Any transport option except <c>applications</c> or +<c>capabilities</c>. +Used as defaults for transport configuration, values passed to +&add_transport; overriding values configured on the service.</p> +</item> + </taglist> <marker id="transport_opt"/> @@ -1061,6 +1083,37 @@ implies having to set matching *-Application-Id AVPs in a </item> <tag> +<marker id="avp_dictionaries"/><c>{avp_dictionaries, [module()]}</c></tag> +<item> +<p> +A list of alternate dictionary modules with which to encode/decode +AVPs that are not defined by the dictionary of the application in +question. +At decode, such AVPs are represented as diameter_avp records in the +<c>'AVP'</c> field of a decoded message or Grouped AVP, the first +alternate that succeeds in decoding the AVP setting the record's value +field. +At encode, values in an <c>'AVP'</c> list can be passed as AVP +name/value 2-tuples, and it is an encode error for no alternate to +define the AVP of such a tuple.</p> + +<p> +Defaults to the empty list.</p> + +<note> +<p> +The motivation for alternate dictionaries is RFC 7683, Diameter +Overload Indication Conveyance (DOIC), which defines AVPs to +be piggybacked onto existing application messages rather than defining +an application of its own. +The DOIC dictionary is provided by the diameter application, as module +<c>diameter_gen_doic_rfc7683</c>, but alternate dictionaries can be +used to encode/decode any set of AVPs not known to an application +dictionary.</p> +</note> +</item> + +<tag> <marker id="capabilities"/><c>{capabilities, [&capability;]}</c></tag> <item> <p> @@ -1075,7 +1128,7 @@ TLS is desired over TCP as implemented by &man_tcp;.</p> </item> <tag> -<marker id="capabilities_cb"/><c>{capabilities_cb, &evaluable;}</c></tag> +<marker id="capabilities_cb"/><c>{capabilities_cb, &eval;}</c></tag> <item> <p> Callback invoked upon reception of CER/CEA during capabilities @@ -1169,7 +1222,7 @@ transport.</p> </item> <tag> -<marker id="disconnect_cb"/><c>{disconnect_cb, &evaluable;}</c></tag> +<marker id="disconnect_cb"/><c>{disconnect_cb, &eval;}</c></tag> <item> <p> Callback invoked prior to terminating the transport process of a @@ -1269,6 +1322,19 @@ Defaults to 5000.</p> </item> <tag> +<marker id="incoming_maxlen"/><c>{incoming_maxlen, 0..16777215}</c></tag> +<item> +<p> +Bound on the expected size of incoming Diameter messages. +Messages larger than the specified number of bytes are discarded.</p> + +<p> +Defaults to <c>16777215</c>, the maximum value of the 24-bit Message +Length field in a Diameter Header.</p> + +</item> + +<tag> <marker id="length_errors"/><c>{length_errors, exit|handle|discard}</c></tag> <item> <p> @@ -1326,7 +1392,64 @@ incoming Diameter request. Options <c>monitor</c> and <c>link</c> are ignored.</p> <p> -Defaults to the list configured on the service if not specified.</p> +Defaults to the empty list.</p> +</item> + +<tag> +<marker id="strict_capx"/><c>{strict_capx, boolean()]}</c></tag> +<item> +<p> +Whether or not to enforce the RFC 6733 requirement that any message +before capabilities exchange should close the peer connection. +If false then unexpected messages are discarded.</p> + +<p> +Defaults to true. +Changing this results in non-standard behaviour, but can be useful in +case peers are known to be behave badly.</p> +</item> + +<tag> +<marker id="strict_mbit"/><c>{strict_mbit, boolean()}</c></tag> +<item> +<p> +Whether or not to regard an AVP setting the M-bit as erroneous when +the command grammar in question does not explicitly allow the AVP. +If <c>true</c> then such AVPs are regarded as 5001 errors, +DIAMETER_AVP_UNSUPPORTED. +If <c>false</c> then the M-bit is ignored and policing +it becomes the receiver's responsibility.</p> + +<p> +Defaults to <c>true</c>.</p> + +<warning> +<p> +RFC 6733 is unclear about the semantics of the M-bit. +One the one hand, the CCF specification in section 3.2 documents AVP +in a command grammar as meaning <em>any</em> arbitrary AVP; on the +other hand, 1.3.4 states that AVPs setting the M-bit cannot be added +to an existing command: the modified command must instead be +placed in a new Diameter application.</p> +<p> +The reason for the latter is presumably interoperability: +allowing arbitrary AVPs setting the M-bit in a command makes its +interpretation implementation-dependent, since there's no +guarantee that all implementations will understand the same set of +arbitrary AVPs in the context of a given command. +However, interpreting <c>AVP</c> in a command grammar as any +AVP, regardless of M-bit, renders 1.3.4 meaningless, since the receiver +can simply ignore any AVP it thinks isn't relevant, regardless of the +sender's intent.</p> +<p> +Beware of confusing mandatory in the sense of the M-bit with mandatory +in the sense of the command grammar. +The former is a semantic requirement: that the receiver understand the +semantics of the AVP in the context in question. +The latter is a syntactic requirement: whether or not the AVP must +occur in the message in question.</p> +</warning> + </item> <tag> diff --git a/lib/diameter/doc/src/diameter_app.xml b/lib/diameter/doc/src/diameter_app.xml index dfcd00975b..aa334beb21 100644 --- a/lib/diameter/doc/src/diameter_app.xml +++ b/lib/diameter/doc/src/diameter_app.xml @@ -13,7 +13,8 @@ <header> <copyright> -<year>2011</year><year>2016</year> +<year>2011</year> +<year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -319,7 +320,7 @@ or &peer_down; callback.</p> <v>Action = Send | Discard | {eval_packet, Action, PostF}</v> <v>Send = {send, &packet; | &message;}</v> <v>Discard = {discard, Reason} | discard</v> -<v>PostF = &mod_evaluable;}</v> +<v>PostF = &mod_eval;}</v> </type> <desc> <p> @@ -371,7 +372,7 @@ discarded}</c>.</p> <v>Action = Send | Discard | {eval_packet, Action, PostF}</v> <v>Send = {send, &packet; | &message;}</v> <v>Discard = {discard, Reason} | discard</v> -<v>PostF = &mod_evaluable;}</v> +<v>PostF = &mod_eval;}</v> </type> <desc> <p> @@ -478,7 +479,7 @@ not selected.</p> | {answer_message, 3000..3999|5000..5999} | {protocol_error, 3000..3999}</v> <v>Opt = &mod_call_opt;</v> -<v>PostF = &mod_evaluable;</v> +<v>PostF = &mod_eval;</v> </type> <desc> <p> diff --git a/lib/diameter/doc/src/diameter_codec.xml b/lib/diameter/doc/src/diameter_codec.xml index 0117c1c88a..5124b49484 100644 --- a/lib/diameter/doc/src/diameter_codec.xml +++ b/lib/diameter/doc/src/diameter_codec.xml @@ -4,7 +4,10 @@ '<seealso marker="diameter_dict#MESSAGE_RECORDS">diameter_dict(4)</seealso>'> <!ENTITY types '<seealso marker="diameter_dict#DATA_TYPES">diameter_dict(4)</seealso>'> - <!ENTITY % also SYSTEM "seealso.ent" > + <!ENTITY decode_format + '<seealso marker="diameter#decode_format">decode format</seealso>'> + +<!ENTITY % also SYSTEM "seealso.ent" > <!ENTITY % here SYSTEM "seehere.ent" > %also; %here; @@ -145,7 +148,8 @@ question.</p> <p> The decoded value of an AVP. Will be <c>undefined</c> on decode if the data bytes could -not be decoded or the AVP is unknown. +not be decoded, the AVP is unknown, or if the &decode_format; is +<c>none</c>. The type of a decoded value is as document in &types;.</p> </item> @@ -230,7 +234,8 @@ header.</p> </item> <tag> -<marker id="message"/><c>message() = record() | list()</c></tag> +<marker id="message"/><c>message() = record() + | maybe_improper_list()</c></tag> <item> <p> The representation of a Diameter message as passed to @@ -240,7 +245,10 @@ a message as defined in a dictionary file is encoded as a record with one field for each component AVP. Equivalently, a message can also be encoded as a list whose head is the atom-valued message name (as specified in the relevant dictionary -file) and whose tail is a list of <c>{AvpName, AvpValue}</c> pairs.</p> +file) and whose tail is either a list of AVP name/values +pairs or a map with values keyed on AVP names. +The format at decode is determined by &mod_decode_format;. +Any of the formats is accepted at encode.</p> <p> Another list-valued representation allows a message to be specified @@ -283,15 +291,16 @@ value other than <c>undefined</c>.</p> <item> <p> The incoming/outgoing message. -For an incoming message, a record if the message can be -decoded in a non-relay application, <c>undefined</c> otherwise. +For an incoming message, a term corresponding to the configured +&decode_format; if the message can be decoded in a non-relay +application, <c>undefined</c> otherwise. For an outgoing message, setting a <c>[&header; | &avp;]</c> list is equivalent to setting the <c>header</c> and <c>avps</c> fields to the corresponding values.</p> <warning> <p> -A record-valued <c>msg</c> field does <em>not</em> imply an absence of +A value in the <c>msg</c> field does <em>not</em> imply an absence of decode errors. The <c>errors</c> field should also be examined.</p> </warning> diff --git a/lib/diameter/doc/src/diameter_sctp.xml b/lib/diameter/doc/src/diameter_sctp.xml index 9b6d629f79..c9b74a9ec5 100644 --- a/lib/diameter/doc/src/diameter_sctp.xml +++ b/lib/diameter/doc/src/diameter_sctp.xml @@ -16,7 +16,7 @@ <header> <copyright> <year>2011</year> -<year>2016</year> +<year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -116,7 +116,6 @@ and port respectively.</p> Multiple <c>ip</c> options can be specified for a multihomed peer. If none are specified then the values of <c>Host-IP-Address</c> in the <c>diameter_service</c> record are used. -(In particular, one of these must be specified.) Option <c>port</c> defaults to 3868 for a listening transport and 0 for a connecting transport.</p> diff --git a/lib/diameter/doc/src/diameter_tcp.xml b/lib/diameter/doc/src/diameter_tcp.xml index 6ca280c52b..1d65d14257 100644 --- a/lib/diameter/doc/src/diameter_tcp.xml +++ b/lib/diameter/doc/src/diameter_tcp.xml @@ -170,14 +170,11 @@ that will not be forthcoming, which will eventually cause the RFC 3539 watchdog to take down the connection.</p> <p> -If an <c>ip</c> option is not specified then the first element of a -non-empty <c>Host-IP-Address</c> list in <c>Svc</c> provides the local -IP address. -If neither is specified then the default address selected by &gen_tcp; -is used. -In all cases, the selected address is either returned from -&start; or passed in a <c>connected</c> message over the transport -interface.</p> +The first element of a non-empty <c>Host-IP-Address</c> list in +<c>Svc</c> provides the local IP address if an <c>ip</c> option is not +specified. +The local address is either returned from&start; or passed in a +<c>connected</c> message over the transport interface.</p> </desc> </func> diff --git a/lib/diameter/doc/src/seealso.ent b/lib/diameter/doc/src/seealso.ent index e5c284c6e8..c5a53670d0 100644 --- a/lib/diameter/doc/src/seealso.ent +++ b/lib/diameter/doc/src/seealso.ent @@ -4,7 +4,7 @@ %CopyrightBegin% -Copyright Ericsson AB 2012-2015. All Rights Reserved. +Copyright Ericsson AB 2012-2017. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -53,7 +53,7 @@ significant. <!ENTITY mod_application_opt '<seealso marker="diameter#application_opt">diameter:application_opt()</seealso>'> <!ENTITY mod_call_opt '<seealso marker="diameter#call_opt">diameter:call_opt()</seealso>'> <!ENTITY mod_capability '<seealso marker="diameter#capability">diameter:capability()</seealso>'> -<!ENTITY mod_evaluable '<seealso marker="diameter#evaluable">diameter:evaluable()</seealso>'> +<!ENTITY mod_eval '<seealso marker="diameter#eval">diameter:eval()</seealso>'> <!ENTITY mod_peer_filter '<seealso marker="diameter#peer_filter">diameter:peer_filter()</seealso>'> <!ENTITY mod_service_event '<seealso marker="diameter#service_event">diameter:service_event()</seealso>'> <!ENTITY mod_service_event_info '<seealso marker="diameter#service_event_info">diameter:service_event_info()</seealso>'> @@ -72,6 +72,7 @@ significant. <!ENTITY watchdog_timer '<seealso marker="#watchdog_timer">watchdog_timer</seealso>'> <!ENTITY mod_string_decode '<seealso marker="diameter#service_opt">diameter:service_opt()</seealso> <seealso marker="diameter#string_decode">string_decode</seealso>'> +<!ENTITY mod_decode_format '<seealso marker="diameter#service_opt">diameter:service_opt()</seealso> <seealso marker="diameter#decode_format">decode_format</seealso>'> <!-- diameter_app --> diff --git a/lib/diameter/doc/standard/rfc7683.txt b/lib/diameter/doc/standard/rfc7683.txt new file mode 100644 index 0000000000..ab2392c6c0 --- /dev/null +++ b/lib/diameter/doc/standard/rfc7683.txt @@ -0,0 +1,2355 @@ + + + + + + +Internet Engineering Task Force (IETF) J. Korhonen, Ed. +Request for Comments: 7683 Broadcom Corporation +Category: Standards Track S. Donovan, Ed. +ISSN: 2070-1721 B. Campbell + Oracle + L. Morand + Orange Labs + October 2015 + + + Diameter Overload Indication Conveyance + +Abstract + + This specification defines a base solution for Diameter overload + control, referred to as Diameter Overload Indication Conveyance + (DOIC). + +Status of This Memo + + This is an Internet Standards Track document. + + This document is a product of the Internet Engineering Task Force + (IETF). It represents the consensus of the IETF community. It has + received public review and has been approved for publication by the + Internet Engineering Steering Group (IESG). Further information on + Internet Standards is available in Section 2 of RFC 5741. + + Information about the current status of this document, any errata, + and how to provide feedback on it may be obtained at + http://www.rfc-editor.org/info/rfc7683. + +Copyright Notice + + Copyright (c) 2015 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + +Korhonen, et al. Standards Track [Page 1] + +RFC 7683 DOIC October 2015 + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Terminology and Abbreviations . . . . . . . . . . . . . . . . 3 + 3. Conventions Used in This Document . . . . . . . . . . . . . . 5 + 4. Solution Overview . . . . . . . . . . . . . . . . . . . . . . 5 + 4.1. Piggybacking . . . . . . . . . . . . . . . . . . . . . . 6 + 4.2. DOIC Capability Announcement . . . . . . . . . . . . . . 7 + 4.3. DOIC Overload Condition Reporting . . . . . . . . . . . . 9 + 4.4. DOIC Extensibility . . . . . . . . . . . . . . . . . . . 11 + 4.5. Simplified Example Architecture . . . . . . . . . . . . . 12 + 5. Solution Procedures . . . . . . . . . . . . . . . . . . . . . 12 + 5.1. Capability Announcement . . . . . . . . . . . . . . . . . 12 + 5.1.1. Reacting Node Behavior . . . . . . . . . . . . . . . 13 + 5.1.2. Reporting Node Behavior . . . . . . . . . . . . . . . 13 + 5.1.3. Agent Behavior . . . . . . . . . . . . . . . . . . . 14 + 5.2. Overload Report Processing . . . . . . . . . . . . . . . 15 + 5.2.1. Overload Control State . . . . . . . . . . . . . . . 15 + 5.2.2. Reacting Node Behavior . . . . . . . . . . . . . . . 19 + 5.2.3. Reporting Node Behavior . . . . . . . . . . . . . . . 20 + 5.3. Protocol Extensibility . . . . . . . . . . . . . . . . . 22 + 6. Loss Algorithm . . . . . . . . . . . . . . . . . . . . . . . 23 + 6.1. Overview . . . . . . . . . . . . . . . . . . . . . . . . 23 + 6.2. Reporting Node Behavior . . . . . . . . . . . . . . . . . 24 + 6.3. Reacting Node Behavior . . . . . . . . . . . . . . . . . 24 + 7. Attribute Value Pairs . . . . . . . . . . . . . . . . . . . . 25 + 7.1. OC-Supported-Features AVP . . . . . . . . . . . . . . . . 25 + 7.2. OC-Feature-Vector AVP . . . . . . . . . . . . . . . . . . 25 + 7.3. OC-OLR AVP . . . . . . . . . . . . . . . . . . . . . . . 26 + 7.4. OC-Sequence-Number AVP . . . . . . . . . . . . . . . . . 26 + 7.5. OC-Validity-Duration AVP . . . . . . . . . . . . . . . . 26 + 7.6. OC-Report-Type AVP . . . . . . . . . . . . . . . . . . . 27 + 7.7. OC-Reduction-Percentage AVP . . . . . . . . . . . . . . . 27 + 7.8. AVP Flag Rules . . . . . . . . . . . . . . . . . . . . . 28 + 8. Error Response Codes . . . . . . . . . . . . . . . . . . . . 28 + 9. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 29 + 9.1. AVP Codes . . . . . . . . . . . . . . . . . . . . . . . . 29 + 9.2. New Registries . . . . . . . . . . . . . . . . . . . . . 29 + 10. Security Considerations . . . . . . . . . . . . . . . . . . . 30 + 10.1. Potential Threat Modes . . . . . . . . . . . . . . . . . 30 + 10.2. Denial-of-Service Attacks . . . . . . . . . . . . . . . 31 + 10.3. Noncompliant Nodes . . . . . . . . . . . . . . . . . . . 32 + 10.4. End-to-End Security Issues . . . . . . . . . . . . . . . 32 + 11. References . . . . . . . . . . . . . . . . . . . . . . . . . 34 + 11.1. Normative References . . . . . . . . . . . . . . . . . . 34 + 11.2. Informative References . . . . . . . . . . . . . . . . . 34 + + + + + +Korhonen, et al. Standards Track [Page 2] + +RFC 7683 DOIC October 2015 + + + Appendix A. Issues Left for Future Specifications . . . . . . . 35 + A.1. Additional Traffic Abatement Algorithms . . . . . . . . . 35 + A.2. Agent Overload . . . . . . . . . . . . . . . . . . . . . 35 + A.3. New Error Diagnostic AVP . . . . . . . . . . . . . . . . 35 + Appendix B. Deployment Considerations . . . . . . . . . . . . . 35 + Appendix C. Considerations for Applications Integrating the DOIC + Solution . . . . . . . . . . . . . . . . . . . . . . 36 + C.1. Application Classification . . . . . . . . . . . . . . . 36 + C.2. Implications of Application Type Overload . . . . . . . . 37 + C.3. Request Transaction Classification . . . . . . . . . . . 38 + C.4. Request Type Overload Implications . . . . . . . . . . . 39 + Contributors . . . . . . . . . . . . . . . . . . . . . . . . . . 41 + Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 42 + +1. Introduction + + This specification defines a base solution for Diameter overload + control, referred to as Diameter Overload Indication Conveyance + (DOIC), based on the requirements identified in [RFC7068]. + + This specification addresses Diameter overload control between + Diameter nodes that support the DOIC solution. The solution, which + is designed to apply to existing and future Diameter applications, + requires no changes to the Diameter base protocol [RFC6733] and is + deployable in environments where some Diameter nodes do not implement + the Diameter overload control solution defined in this specification. + + A new application specification can incorporate the overload control + mechanism specified in this document by making it mandatory to + implement for the application and referencing this specification + normatively. It is the responsibility of the Diameter application + designers to define how overload control mechanisms work on that + application. + + Note that the overload control solution defined in this specification + does not address all the requirements listed in [RFC7068]. A number + of features related to overload control are left for future + specifications. See Appendix A for a list of extensions that are + currently being considered. + +2. Terminology and Abbreviations + + Abatement + + Reaction to receipt of an overload report resulting in a reduction + in traffic sent to the reporting node. Abatement actions include + diversion and throttling. + + + + +Korhonen, et al. Standards Track [Page 3] + +RFC 7683 DOIC October 2015 + + + Abatement Algorithm + + An extensible method requested by reporting nodes and used by + reacting nodes to reduce the amount of traffic sent during an + occurrence of overload control. + + Diversion + + An overload abatement treatment where the reacting node selects + alternate destinations or paths for requests. + + Host-Routed Requests + + Requests that a reacting node knows will be served by a particular + host, either due to the presence of a Destination-Host Attribute + Value Pair (AVP) or by some other local knowledge on the part of + the reacting node. + + Overload Control State (OCS) + + Internal state maintained by a reporting or reacting node + describing occurrences of overload control. + + Overload Report (OLR) + + Overload control information for a particular overload occurrence + sent by a reporting node. + + Reacting Node + + A Diameter node that acts upon an overload report. + + Realm-Routed Requests + + Requests sent by a reacting node where the reacting node does not + know to which host the request will be routed. + + Reporting Node + + A Diameter node that generates an overload report. (This may or + may not be the overloaded node.) + + + + + + + + + + +Korhonen, et al. Standards Track [Page 4] + +RFC 7683 DOIC October 2015 + + + Throttling + + An abatement treatment that limits the number of requests sent by + the reacting node. Throttling can include a Diameter Client + choosing to not send requests, or a Diameter Agent or Server + rejecting requests with appropriate error responses. In both + cases, the result of the throttling is a permanent rejection of + the transaction. + +3. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in RFC 2119 [RFC2119]. + + The interpretation from RFC 2119 [RFC2119] does not apply for the + above listed words when they are not used in all caps. + +4. Solution Overview + + The Diameter Overload Information Conveyance (DOIC) solution allows + Diameter nodes to request that other Diameter nodes perform overload + abatement actions, that is, actions to reduce the load offered to the + overloaded node or realm. + + A Diameter node that supports DOIC is known as a "DOIC node". Any + Diameter node can act as a DOIC node, including Diameter Clients, + Diameter Servers, and Diameter Agents. DOIC nodes are further + divided into "Reporting Nodes" and "Reacting Nodes." A reporting + node requests overload abatement by sending Overload Reports (OLRs). + + A reacting node acts upon OLRs and performs whatever actions are + needed to fulfill the abatement requests included in the OLRs. A + reporting node may report overload on its own behalf or on behalf of + other nodes. Likewise, a reacting node may perform overload + abatement on its own behalf or on behalf of other nodes. + + A Diameter node's role as a DOIC node is independent of its Diameter + role. For example, Diameter Agents may act as DOIC nodes, even + though they are not endpoints in the Diameter sense. Since Diameter + enables bidirectional applications, where Diameter Servers can send + requests towards Diameter Clients, a given Diameter node can + simultaneously act as both a reporting node and a reacting node. + + Likewise, a Diameter Agent may act as a reacting node from the + perspective of upstream nodes, and a reporting node from the + perspective of downstream nodes. + + + + +Korhonen, et al. Standards Track [Page 5] + +RFC 7683 DOIC October 2015 + + + DOIC nodes do not generate new messages to carry DOIC-related + information. Rather, they "piggyback" DOIC information over existing + Diameter messages by inserting new AVPs into existing Diameter + requests and responses. Nodes indicate support for DOIC, and any + needed DOIC parameters, by inserting an OC-Supported-Features AVP + (Section 7.1) into existing requests and responses. Reporting nodes + send OLRs by inserting OC-OLR AVPs (Section 7.3). + + A given OLR applies to the Diameter realm and application of the + Diameter message that carries it. If a reporting node supports more + than one realm and/or application, it reports independently for each + combination of realm and application. Similarly, the OC-Supported- + Features AVP applies to the realm and application of the enclosing + message. This implies that a node may support DOIC for one + application and/or realm, but not another, and may indicate different + DOIC parameters for each application and realm for which it supports + DOIC. + + Reacting nodes perform overload abatement according to an agreed-upon + abatement algorithm. An abatement algorithm defines the meaning of + some of the parameters of an OLR and the procedures required for + overload abatement. An overload abatement algorithm separates + Diameter requests into two sets. The first set contains the requests + that are to undergo overload abatement treatment of either throttling + or diversion. The second set contains the requests that are to be + given normal routing treatment. This document specifies a single + "must-support" algorithm, namely, the "loss" algorithm (Section 6). + Future specifications may introduce new algorithms. + + Overload conditions may vary in scope. For example, a single + Diameter node may be overloaded, in which case, reacting nodes may + attempt to send requests to other destinations. On the other hand, + an entire Diameter realm may be overloaded, in which case, such + attempts would do harm. DOIC OLRs have a concept of "report type" + (Section 7.6), where the type defines such behaviors. Report types + are extensible. This document defines report types for overload of a + specific host and for overload of an entire realm. + + DOIC works through non-supporting Diameter Agents that properly pass + unknown AVPs unchanged. + +4.1. Piggybacking + + There is no new Diameter application defined to carry overload- + related AVPs. The overload control AVPs defined in this + specification have been designed to be piggybacked on top of existing + + + + + +Korhonen, et al. Standards Track [Page 6] + +RFC 7683 DOIC October 2015 + + + application messages. This is made possible by adding the optional + overload control AVPs OC-OLR and OC-Supported-Features into existing + commands. + + Reacting nodes indicate support for DOIC by including the + OC-Supported-Features AVP in all request messages originated or + relayed by the reacting node. + + Reporting nodes indicate support for DOIC by including the + OC-Supported-Features AVP in all answer messages that are originated + or relayed by the reporting node and that are in response to a + request that contained the OC-Supported-Features AVP. Reporting + nodes may include overload reports using the OC-OLR AVP in answer + messages. + + Note that the overload control solution does not have fixed server + and client roles. The DOIC node role is determined based on the + message type: whether the message is a request (i.e., sent by a + "reacting node") or an answer (i.e., sent by a "reporting node"). + Therefore, in a typical client-server deployment, the Diameter Client + may report its overload condition to the Diameter Server for any + Diameter-Server-initiated message exchange. An example of such is + the Diameter Server requesting a re-authentication from a Diameter + Client. + +4.2. DOIC Capability Announcement + + The DOIC solution supports the ability for Diameter nodes to + determine if other nodes in the path of a request support the + solution. This capability is referred to as DOIC Capability + Announcement (DCA) and is separate from the Diameter Capability + Exchange. + + The DCA mechanism uses the OC-Supported-Features AVPs to indicate the + Diameter overload features supported. + + The first node in the path of a Diameter request that supports the + DOIC solution inserts the OC-Supported-Features AVP in the request + message. + + The individual features supported by the DOIC nodes are indicated in + the OC-Feature-Vector AVP. Any semantics associated with the + features will be defined in extension specifications that introduce + the features. + + Note: As discussed elsewhere in the document, agents in the path + of the request can modify the OC-Supported-Features AVP. + + + + +Korhonen, et al. Standards Track [Page 7] + +RFC 7683 DOIC October 2015 + + + Note: The DOIC solution must support deployments where Diameter + Clients and/or Diameter Servers do not support the DOIC solution. + In this scenario, Diameter Agents that support the DOIC solution + may handle overload abatement for the non-supporting Diameter + nodes. In this case, the DOIC agent will insert the OC-Supported- + Features AVP in requests that do not already contain one, telling + the reporting node that there is a DOIC node that will handle + overload abatement. For transactions where there was an + OC-Supporting-Features AVP in the request, the agent will insert + the OC-Supported-Features AVP in answers, telling the reacting + node that there is a reporting node. + + The OC-Feature-Vector AVP will always contain an indication of + support for the loss overload abatement algorithm defined in this + specification (see Section 6). This ensures that a reporting node + always supports at least one of the advertised abatement algorithms + received in a request messages. + + The reporting node inserts the OC-Supported-Features AVP in all + answer messages to requests that contained the OC-Supported-Features + AVP. The contents of the reporting node's OC-Supported-Features AVP + indicate the set of Diameter overload features supported by the + reporting node. This specification defines one exception -- the + reporting node only includes an indication of support for one + overload abatement algorithm, independent of the number of overload + abatement algorithms actually supported by the reacting node. The + overload abatement algorithm indicated is the algorithm that the + reporting node intends to use should it enter an overload condition. + Reacting nodes can use the indicated overload abatement algorithm to + prepare for possible overload reports and must use the indicated + overload abatement algorithm if traffic reduction is actually + requested. + + Note that the loss algorithm defined in this document is a + stateless abatement algorithm. As a result, it does not require + any actions by reacting nodes prior to the receipt of an overload + report. Stateful abatement algorithms that base the abatement + logic on a history of request messages sent might require reacting + nodes to maintain state in advance of receiving an overload report + to ensure that the overload reports can be properly handled. + + While it should only be done in exceptional circumstances and not + during an active occurrence of overload, a reacting node that wishes + to transition to a different abatement algorithm can stop advertising + support for the algorithm indicated by the reporting node, as long as + support for the loss algorithm is always advertised. + + + + + +Korhonen, et al. Standards Track [Page 8] + +RFC 7683 DOIC October 2015 + + + The DCA mechanism must also allow the scenario where the set of + features supported by the sender of a request and by agents in the + path of a request differ. In this case, the agent can update the + OC-Supported-Features AVP to reflect the mixture of the two sets of + supported features. + + Note: The logic to determine if the content of the OC-Supported- + Features AVP should be changed is out of scope for this document, + as is the logic to determine the content of a modified + OC-Supported-Features AVP. These are left to implementation + decisions. Care must be taken not to introduce interoperability + issues for downstream or upstream DOIC nodes. As such, the agent + must act as a fully compliant reporting node to the downstream + reacting node and as a fully compliant reacting node to the + upstream reporting node. + +4.3. DOIC Overload Condition Reporting + + As with DOIC capability announcement, overload condition reporting + uses new AVPs (Section 7.3) to indicate an overload condition. + + The OC-OLR AVP is referred to as an overload report. The OC-OLR AVP + includes the type of report, a sequence number, the length of time + that the report is valid, and AVPs specific to the abatement + algorithm. + + Two types of overload reports are defined in this document: host + reports and realm reports. + + A report of type "HOST_REPORT" is sent to indicate the overload of a + specific host, identified by the Origin-Host AVP of the message + containing the OLR, for the Application-ID indicated in the + transaction. When receiving an OLR of type "HOST_REPORT", a reacting + node applies overload abatement treatment to the host-routed requests + identified by the overload abatement algorithm (as defined in + Section 2) sent for this application to the overloaded host. + + A report of type "REALM_REPORT" is sent to indicate the overload of a + realm for the Application-ID indicated in the transaction. The + overloaded realm is identified by the Destination-Realm AVP of the + message containing the OLR. When receiving an OLR of type + "REALM_REPORT", a reacting node applies overload abatement treatment + to realm-routed requests identified by the overload abatement + algorithm (as defined in Section 2) sent for this application to the + overloaded realm. + + + + + + +Korhonen, et al. Standards Track [Page 9] + +RFC 7683 DOIC October 2015 + + + This document assumes that there is a single source for realm reports + for a given realm, or that if multiple nodes can send realm reports, + that each such node has full knowledge of the overload state of the + entire realm. A reacting node cannot distinguish between receiving + realm reports from a single node or from multiple nodes. + + Note: Known issues exist if there are multiple sources for + overload reports that apply to the same Diameter entity. Reacting + nodes have no way of determining the source and, as such, will + treat them as coming from a single source. Variance in sequence + numbers between the two sources can then cause incorrect overload + abatement treatment to be applied for indeterminate periods of + time. + + Reporting nodes are responsible for determining the need for a + reduction of traffic. The method for making this determination is + implementation specific and depends on the type of overload report + being generated. A host report might be generated by tracking use of + resources required by the host to handle transactions for the + Diameter application. A realm report generally impacts the traffic + sent to multiple hosts and, as such, requires tracking the capacity + of all servers able to handle realm-routed requests for the + application and realm. + + Once a reporting node determines the need for a reduction in traffic, + it uses the DOIC-defined AVPs to report on the condition. These AVPs + are included in answer messages sent or relayed by the reporting + node. The reporting node indicates the overload abatement algorithm + that is to be used to handle the traffic reduction in the + OC-Supported-Features AVP. The OC-OLR AVP is used to communicate + information about the requested reduction. + + Reacting nodes, upon receipt of an overload report, apply the + overload abatement algorithm to traffic impacted by the overload + report. The method used to determine the requests that are to + receive overload abatement treatment is dependent on the abatement + algorithm. The loss abatement algorithm is defined in this document + (Section 6). Other abatement algorithms can be defined in extensions + to the DOIC solution. + + Two types of overload abatement treatment are defined, diversion and + throttling. Reacting nodes are responsible for determining which + treatment is appropriate for individual requests. + + As the conditions that lead to the generation of the overload report + change, the reporting node can send new overload reports requesting + greater reduction if the condition gets worse or less reduction if + the condition improves. The reporting node sends an overload report + + + +Korhonen, et al. Standards Track [Page 10] + +RFC 7683 DOIC October 2015 + + + with a duration of zero to indicate that the overload condition has + ended and abatement is no longer needed. + + The reacting node also determines when the overload report expires + based on the OC-Validity-Duration AVP in the overload report and + stops applying the abatement algorithm when the report expires. + + Note that erroneous overload reports can be used for DoS attacks. + This includes the ability to indicate that a significant reduction in + traffic, up to and including a request for no traffic, should be sent + to a reporting node. As such, care should be taken to verify the + sender of overload reports. + +4.4. DOIC Extensibility + + The DOIC solution is designed to be extensible. This extensibility + is based on existing Diameter-based extensibility mechanisms, along + with the DOIC capability announcement mechanism. + + There are multiple categories of extensions that are expected. This + includes the definition of new overload abatement algorithms, the + definition of new report types, and the definition of new scopes of + messages impacted by an overload report. + + A DOIC node communicates supported features by including them in the + OC-Feature-Vector AVP, as a sub-AVP of OC-Supported-Features. Any + non-backwards-compatible DOIC extensions define new values for the + OC-Feature-Vector AVP. DOIC extensions also have the ability to add + new AVPs to the OC-Supported-Features AVP, if additional information + about the new feature is required. + + Overload reports can also be extended by adding new sub-AVPs to the + OC-OLR AVP, allowing reporting nodes to communicate additional + information about handling an overload condition. + + If necessary, new extensions can also define new AVPs that are not + part of the OC-Supported-Features and OC-OLR group AVPs. It is, + however, recommended that DOIC extensions use the OC-Supported- + Features AVP and OC-OLR AVP to carry all DOIC-related AVPs. + + + + + + + + + + + + +Korhonen, et al. Standards Track [Page 11] + +RFC 7683 DOIC October 2015 + + +4.5. Simplified Example Architecture + + Figure 1 illustrates the simplified architecture for Diameter + overload information conveyance. + + Realm X Same or other Realms + <--------------------------------------> <----------------------> + + + +--------+ : (optional) : + |Diameter| : : + |Server A|--+ .--. : +--------+ : .--. + +--------+ | _( `. : |Diameter| : _( `. +--------+ + +--( )--:-| Agent |-:--( )--|Diameter| + +--------+ | ( ` . ) ) : +--------+ : ( ` . ) ) | Client | + |Diameter|--+ `--(___.-' : : `--(___.-' +--------+ + |Server B| : : + +--------+ : : + + End-to-end Overload Indication + 1) <-----------------------------------------------> + Diameter Application Y + + Overload Indication A Overload Indication A' + 2) <----------------------> <----------------------> + Diameter Application Y Diameter Application Y + + Figure 1: Simplified Architecture Choices for Overload Indication + Delivery + + In Figure 1, the Diameter overload indication can be conveyed (1) + end-to-end between servers and clients or (2) between servers and the + Diameter Agent inside the realm and then between the Diameter Agent + and the clients. + +5. Solution Procedures + + This section outlines the normative behavior for the DOIC solution. + +5.1. Capability Announcement + + This section defines DOIC Capability Announcement (DCA) behavior. + + Note: This specification assumes that changes in DOIC node + capabilities are relatively rare events that occur as a result of + administrative action. Reacting nodes ought to minimize changes + that force the reporting node to change the features being used, + especially during active overload conditions. But even if + + + +Korhonen, et al. Standards Track [Page 12] + +RFC 7683 DOIC October 2015 + + + reacting nodes avoid such changes, reporting nodes still have to + be prepared for them to occur. For example, differing + capabilities between multiple reacting nodes may still force a + reporting node to select different features on a per-transaction + basis. + +5.1.1. Reacting Node Behavior + + A reacting node MUST include the OC-Supported-Features AVP in all + requests. It MAY include the OC-Feature-Vector AVP, as a sub-AVP of + OC-Supported-Features. If it does so, it MUST indicate support for + the "loss" algorithm. If the reacting node is configured to support + features (including other algorithms) in addition to the loss + algorithm, it MUST indicate such support in an OC-Feature-Vector AVP. + + An OC-Supported-Features AVP in answer messages indicates there is a + reporting node for the transaction. The reacting node MAY take + action, for example, creating state for some stateful abatement + algorithm, based on the features indicated in the OC-Feature-Vector + AVP. + + Note: The loss abatement algorithm does not require stateful + behavior when there is no active overload report. + + Reacting nodes need to be prepared for the reporting node to change + selected algorithms. This can happen at any time, including when the + reporting node has sent an active overload report. The reacting node + can minimize the potential for changes by modifying the advertised + abatement algorithms sent to an overloaded reporting node to the + currently selected algorithm and loss (or just loss if it is the + currently selected algorithm). This has the effect of limiting the + potential change in abatement algorithm from the currently selected + algorithm to loss, avoiding changes to more complex abatement + algorithms that require state to operate properly. + +5.1.2. Reporting Node Behavior + + Upon receipt of a request message, a reporting node determines if + there is a reacting node for the transaction based on the presence of + the OC-Supported-Features AVP in the request message. + + If the request message contains an OC-Supported-Features AVP, then a + reporting node MUST include the OC-Supported-Features AVP in the + answer message for that transaction. + + Note: Capability announcement is done on a per-transaction basis. + The reporting node cannot assume that the capabilities announced + by a reacting node will be the same between transactions. + + + +Korhonen, et al. Standards Track [Page 13] + +RFC 7683 DOIC October 2015 + + + A reporting node MUST NOT include the OC-Supported-Features AVP, + OC-OLR AVP, or any other overload control AVPs defined in extension + documents in response messages for transactions where the request + message does not include the OC-Supported-Features AVP. Lack of the + OC-Supported-Features AVP in the request message indicates that there + is no reacting node for the transaction. + + A reporting node knows what overload control functionality is + supported by the reacting node based on the content or absence of the + OC-Feature-Vector AVP within the OC-Supported-Features AVP in the + request message. + + A reporting node MUST select a single abatement algorithm in the + OC-Feature-Vector AVP. The abatement algorithm selected MUST + indicate the abatement algorithm the reporting node wants the + reacting node to use when the reporting node enters an overload + condition. + + The abatement algorithm selected MUST be from the set of abatement + algorithms contained in the request message's OC-Feature-Vector AVP. + + A reporting node that selects the loss algorithm may do so by + including the OC-Feature-Vector AVP with an explicit indication of + the loss algorithm, or it MAY omit the OC-Feature-Vector AVP. If it + selects a different algorithm, it MUST include the OC-Feature-Vector + AVP with an explicit indication of the selected algorithm. + + The reporting node SHOULD indicate support for other DOIC features + defined in extension documents that it supports and that apply to the + transaction. It does so using the OC-Feature-Vector AVP. + + Note: Not all DOIC features will apply to all Diameter + applications or deployment scenarios. The features included in + the OC-Feature-Vector AVP are based on local policy of the + reporting node. + +5.1.3. Agent Behavior + + Diameter Agents that support DOIC can ensure that all messages + relayed by the agent contain the OC-Supported-Features AVP. + + A Diameter Agent MAY take on reacting node behavior for Diameter + endpoints that do not support the DOIC solution. A Diameter Agent + detects that a Diameter endpoint does not support DOIC reacting node + behavior when there is no OC-Supported-Features AVP in a request + message. + + + + + +Korhonen, et al. Standards Track [Page 14] + +RFC 7683 DOIC October 2015 + + + For a Diameter Agent to be a reacting node for a non-supporting + Diameter endpoint, the Diameter Agent MUST include the OC-Supported- + Features AVP in request messages it relays that do not contain the + OC-Supported-Features AVP. + + A Diameter Agent MAY take on reporting node behavior for Diameter + endpoints that do not support the DOIC solution. The Diameter Agent + MUST have visibility to all traffic destined for the non-supporting + host in order to become the reporting node for the Diameter endpoint. + A Diameter Agent detects that a Diameter endpoint does not support + DOIC reporting node behavior when there is no OC-Supported-Features + AVP in an answer message for a transaction that contained the + OC-Supported-Features AVP in the request message. + + If a request already has the OC-Supported-Features AVP, a Diameter + Agent MAY modify it to reflect the features appropriate for the + transaction. Otherwise, the agent relays the OC-Supported-Features + AVP without change. + + Example: If the agent supports a superset of the features reported + by the reacting node, then the agent might choose, based on local + policy, to advertise that superset of features to the reporting + node. + + If the Diameter Agent changes the OC-Supported-Features AVP in a + request message, then it is likely it will also need to modify the + OC-Supported-Features AVP in the answer message for the transaction. + A Diameter Agent MAY modify the OC-Supported-Features AVP carried in + answer messages. + + When making changes to the OC-Supported-Features or OC-OLR AVPs, the + Diameter Agent needs to ensure consistency in its behavior with both + upstream and downstream DOIC nodes. + +5.2. Overload Report Processing + +5.2.1. Overload Control State + + Both reacting and reporting nodes maintain Overload Control State + (OCS) for active overload conditions. The following sections define + behavior associated with that OCS. + + The contents of the OCS in the reporting node and in the reacting + node represent logical constructs. The actual internal physical + structure of the state included in the OCS is an implementation + decision. + + + + + +Korhonen, et al. Standards Track [Page 15] + +RFC 7683 DOIC October 2015 + + +5.2.1.1. Overload Control State for Reacting Nodes + + A reacting node maintains the following OCS per supported Diameter + application: + + o a host-type OCS entry for each Destination-Host to which it sends + host-type requests and + + o a realm-type OCS entry for each Destination-Realm to which it + sends realm-type requests. + + A host-type OCS entry is identified by the pair of Application-ID and + the node's DiameterIdentity. + + A realm-type OCS entry is identified by the pair of Application-ID + and realm. + + The host-type and realm-type OCS entries include the following + information (the actual information stored is an implementation + decision): + + o Sequence number (as received in OC-OLR; see Section 7.3) + + o Time of expiry (derived from OC-Validity-Duration AVP received in + the OC-OLR AVP and time of reception of the message carrying + OC-OLR AVP) + + o Selected abatement algorithm (as received in the OC-Supported- + Features AVP) + + o Input data that is abatement algorithm specific (as received in + the OC-OLR AVP -- for example, OC-Reduction-Percentage for the + loss abatement algorithm) + +5.2.1.2. Overload Control State for Reporting Nodes + + A reporting node maintains OCS entries per supported Diameter + application, per supported (and eventually selected) abatement + algorithm, and per report type. + + An OCS entry is identified by the tuple of Application-ID, report + type, and abatement algorithm, and it includes the following + information (the actual information stored is an implementation + decision): + + o Sequence number + + o Validity duration + + + +Korhonen, et al. Standards Track [Page 16] + +RFC 7683 DOIC October 2015 + + + o Expiration time + + o Input data that is algorithm specific (for example, the reduction + percentage for the loss abatement algorithm) + +5.2.1.3. Reacting Node's Maintenance of Overload Control State + + When a reacting node receives an OC-OLR AVP, it MUST determine if it + is for an existing or new overload condition. + + Note: For the remainder of this section, the term "OLR" refers to + the combination of the contents of the received OC-OLR AVP and the + abatement algorithm indicated in the received OC-Supported- + Features AVP. + + When receiving an answer message with multiple OLRs of different + supported report types, a reacting node MUST process each received + OLR. + + The OLR is for an existing overload condition if a reacting node has + an OCS that matches the received OLR. + + For a host report, this means it matches the Application-ID and the + host's DiameterIdentity in an existing host OCS entry. + + For a realm report, this means it matches the Application-ID and the + realm in an existing realm OCS entry. + + If the OLR is for an existing overload condition, then a reacting + node MUST determine if the OLR is a retransmission or an update to + the existing OLR. + + If the sequence number for the received OLR is greater than the + sequence number stored in the matching OCS entry, then a reacting + node MUST update the matching OCS entry. + + If the sequence number for the received OLR is less than or equal to + the sequence number in the matching OCS entry, then a reacting node + MUST silently ignore the received OLR. The matching OCS MUST NOT be + updated in this case. + + If the reacting node determines that the sequence number has rolled + over, then the reacting node MUST update the matching OCS entry. + This can be determined by recognizing that the number has changed + from a value within 1% of the maximum value in the OC-Sequence-Number + AVP to a value within 1% of the minimum value in the OC-Sequence- + Number AVP. + + + + +Korhonen, et al. Standards Track [Page 17] + +RFC 7683 DOIC October 2015 + + + If the received OLR is for a new overload condition, then a reacting + node MUST generate a new OCS entry for the overload condition. + + For a host report, this means a reacting node creates an OCS entry + with the Application-ID in the received message and DiameterIdentity + of the Origin-Host in the received message. + + Note: This solution assumes that the Origin-Host AVP in the answer + message included by the reporting node is not changed along the + path to the reacting node. + + For a realm report, this means a reacting node creates an OCS entry + with the Application-ID in the received message and realm of the + Origin-Realm in the received message. + + If the received OLR contains a validity duration of zero ("0"), then + a reacting node MUST update the OCS entry as being expired. + + Note: It is not necessarily appropriate to delete the OCS entry, + as the recommended behavior is that the reacting node slowly + returns to full traffic when ending an overload abatement period. + + The reacting node does not delete an OCS when receiving an answer + message that does not contain an OC-OLR AVP (i.e., absence of OLR + means "no change"). + +5.2.1.4. Reporting Node's Maintenance of Overload Control State + + A reporting node SHOULD create a new OCS entry when entering an + overload condition. + + Note: If a reporting node knows through absence of the + OC-Supported-Features AVP in received messages that there are no + reacting nodes supporting DOIC, then the reporting node can choose + to not create OCS entries. + + When generating a new OCS entry, the sequence number SHOULD be set to + zero ("0"). + + When generating sequence numbers for new overload conditions, the new + sequence number MUST be greater than any sequence number in an active + (unexpired) overload report for the same application and report type + previously sent by the reporting node. This property MUST hold over + a reboot of the reporting node. + + + + + + + +Korhonen, et al. Standards Track [Page 18] + +RFC 7683 DOIC October 2015 + + + Note: One way of addressing this over a reboot of a reporting node + is to use a timestamp for the first overload condition that occurs + after the report and to start using sequences beginning with zero + for subsequent overload conditions. + + A reporting node MUST update an OCS entry when it needs to adjust the + validity duration of the overload condition at reacting nodes. + + Example: If a reporting node wishes to instruct reacting nodes to + continue overload abatement for a longer period of time than + originally communicated. This also applies if the reporting node + wishes to shorten the period of time that overload abatement is to + continue. + + A reporting node MUST update an OCS entry when it wishes to adjust + any parameters specific to the abatement algorithm, including, for + example, the reduction percentage used for the loss abatement + algorithm. + + Example: If a reporting node wishes to change the reduction + percentage either higher (if the overload condition has worsened) + or lower (if the overload condition has improved), then the + reporting node would update the appropriate OCS entry. + + A reporting node MUST increment the sequence number associated with + the OCS entry anytime the contents of the OCS entry are changed. + This will result in a new sequence number being sent to reacting + nodes, instructing them to process the OC-OLR AVP. + + A reporting node SHOULD update an OCS entry with a validity duration + of zero ("0") when the overload condition ends. + + Note: If a reporting node knows that the OCS entries in the + reacting nodes are near expiration, then the reporting node might + decide not to send an OLR with a validity duration of zero. + + A reporting node MUST keep an OCS entry with a validity duration of + zero ("0") for a period of time long enough to ensure that any + unexpired reacting node's OCS entry created as a result of the + overload condition in the reporting node is deleted. + +5.2.2. Reacting Node Behavior + + When a reacting node sends a request, it MUST determine if that + request matches an active OCS. + + + + + + +Korhonen, et al. Standards Track [Page 19] + +RFC 7683 DOIC October 2015 + + + If the request matches an active OCS, then the reacting node MUST use + the overload abatement algorithm indicated in the OCS to determine if + the request is to receive overload abatement treatment. + + For the loss abatement algorithm defined in this specification, see + Section 6 for the overload abatement algorithm logic applied. + + If the overload abatement algorithm selects the request for overload + abatement treatment, then the reacting node MUST apply overload + abatement treatment on the request. The abatement treatment applied + depends on the context of the request. + + If diversion abatement treatment is possible (i.e., a different path + for the request can be selected where the overloaded node is not part + of the different path), then the reacting node SHOULD apply diversion + abatement treatment to the request. The reacting node MUST apply + throttling abatement treatment to requests identified for abatement + treatment when diversion treatment is not possible or was not + applied. + + Note: This only addresses the case where there are two defined + abatement treatments, diversion and throttling. Any extension + that defines a new abatement treatment must also define its + interaction with existing treatments. + + If the overload abatement treatment results in throttling of the + request and if the reacting node is an agent, then the agent MUST + send an appropriate error as defined in Section 8. + + Diameter endpoints that throttle requests need to do so according to + the rules of the client application. Those rules will vary by + application and are beyond the scope of this document. + + In the case that the OCS entry indicated no traffic was to be sent to + the overloaded entity and the validity duration expires, then + overload abatement associated with the overload report MUST be ended + in a controlled fashion. + +5.2.3. Reporting Node Behavior + + If there is an active OCS entry, then a reporting node SHOULD include + the OC-OLR AVP in all answers to requests that contain the + OC-Supported-Features AVP and that match the active OCS entry. + + Note: A request matches 1) if the Application-ID in the request + matches the Application-ID in any active OCS entry and 2) if the + report type in the OCS entry matches a report type supported by + the reporting node as indicated in the OC-Supported-Features AVP. + + + +Korhonen, et al. Standards Track [Page 20] + +RFC 7683 DOIC October 2015 + + + The contents of the OC-OLR AVP depend on the selected algorithm. + + A reporting node MAY choose to not resend an overload report to a + reacting node if it can guarantee that this overload report is + already active in the reacting node. + + Note: In some cases (e.g., when there are one or more agents in + the path between reporting and reacting nodes, or when overload + reports are discarded by reacting nodes), a reporting node may not + be able to guarantee that the reacting node has received the + report. + + A reporting node MUST NOT send overload reports of a type that has + not been advertised as supported by the reacting node. + + Note: A reacting node implicitly advertises support for the host + and realm report types by including the OC-Supported-Features AVP + in the request. Support for other report types will be explicitly + indicated by new feature bits in the OC-Feature-Vector AVP. + + A reporting node SHOULD explicitly indicate the end of an overload + occurrence by sending a new OLR with OC-Validity-Duration set to a + value of zero ("0"). The reporting node SHOULD ensure that all + reacting nodes receive the updated overload report. + + A reporting node MAY rely on the OC-Validity-Duration AVP values for + the implicit cleanup of overload control state on the reacting node. + + Note: All OLRs sent have an expiration time calculated by adding + the validity duration contained in the OLR to the time the message + was sent. Transit time for the OLR can be safely ignored. The + reporting node can ensure that all reacting nodes have received + the OLR by continuing to send it in answer messages until the + expiration time for all OLRs sent for that overload condition have + expired. + + When a reporting node sends an OLR, it effectively delegates any + necessary throttling to downstream nodes. If the reporting node also + locally throttles the same set of messages, the overall number of + throttled requests may be higher than intended. Therefore, before + applying local message throttling, a reporting node needs to check if + these messages match existing OCS entries, indicating that these + messages have survived throttling applied by downstream nodes that + have received the related OLR. + + However, even if the set of messages match existing OCS entries, the + reporting node can still apply other abatement methods such as + diversion. The reporting node might also need to throttle requests + + + +Korhonen, et al. Standards Track [Page 21] + +RFC 7683 DOIC October 2015 + + + for reasons other than overload. For example, an agent or server + might have a configured rate limit for each client and might throttle + requests that exceed that limit, even if such requests had already + been candidates for throttling by downstream nodes. The reporting + node also has the option to send new OLRs requesting greater + reductions in traffic, reducing the need for local throttling. + + A reporting node SHOULD decrease requested overload abatement + treatment in a controlled fashion to avoid oscillations in traffic. + + Example: A reporting node might wait some period of time after + overload ends before terminating the OLR, or it might send a + series of OLRs indicating progressively less overload severity. + +5.3. Protocol Extensibility + + The DOIC solution can be extended. Types of potential extensions + include new traffic abatement algorithms, new report types, or other + new functionality. + + When defining a new extension that requires new normative behavior, + the specification must define a new feature for the OC-Feature-Vector + AVP. This feature bit is used to communicate support for the new + feature. + + The extension may define new AVPs for use in the DOIC Capability + Announcement and for use in DOIC overload reporting. These new AVPs + SHOULD be defined to be extensions to the OC-Supported-Features or + OC-OLR AVPs defined in this document. + + The Grouped AVP extension mechanisms defined in [RFC6733] apply. + This allows, for example, defining a new feature that is mandatory to + be understood even when piggybacked on an existing application. + + When defining new report type values, the corresponding specification + must define the semantics of the new report types and how they affect + the OC-OLR AVP handling. + + The OC-Supported-Feature and OC-OLR AVPs can be expanded with + optional sub-AVPs only if a legacy DOIC implementation can safely + ignore them without breaking backward compatibility for the given + OC-Report-Type AVP value. Any new sub-AVPs must not require that the + M-bit be set. + + Documents that introduce new report types must describe any + limitations on their use across non-supporting agents. + + + + + +Korhonen, et al. Standards Track [Page 22] + +RFC 7683 DOIC October 2015 + + + As with any Diameter specification, RFC 6733 requires all new AVPs to + be registered with IANA. See Section 9 for the required procedures. + New features (feature bits in the OC-Feature-Vector AVP) and report + types (in the OC-Report-Type AVP) MUST be registered with IANA. + +6. Loss Algorithm + + This section documents the Diameter overload loss abatement + algorithm. + +6.1. Overview + + The DOIC specification supports the ability for multiple overload + abatement algorithms to be specified. The abatement algorithm used + for any instance of overload is determined by the DOIC Capability + Announcement process documented in Section 5.1. + + The loss algorithm described in this section is the default algorithm + that must be supported by all Diameter nodes that support DOIC. + + The loss algorithm is designed to be a straightforward and stateless + overload abatement algorithm. It is used by reporting nodes to + request a percentage reduction in the amount of traffic sent. The + traffic impacted by the requested reduction depends on the type of + overload report. + + Reporting nodes request the stateless reduction of the number of + requests by an indicated percentage. This percentage reduction is in + comparison to the number of messages the node otherwise would send, + regardless of how many requests the node might have sent in the past. + + From a conceptual level, the logic at the reacting node could be + outlined as follows. + + 1. An overload report is received, and the associated OCS is either + saved or updated (if required) by the reacting node. + + 2. A new Diameter request is generated by the application running on + the reacting node. + + 3. The reacting node determines that an active overload report + applies to the request, as indicated by the corresponding OCS + entry. + + 4. The reacting node determines if overload abatement treatment + should be applied to the request. One approach that could be + taken for each request is to select a uniformly selected random + number between 1 and 100. If the random number is less than or + + + +Korhonen, et al. Standards Track [Page 23] + +RFC 7683 DOIC October 2015 + + + equal to the indicated reduction percentage, then the request is + given abatement treatment; otherwise, the request is given normal + routing treatment. + +6.2. Reporting Node Behavior + + The method a reporting node uses to determine the amount of traffic + reduction required to address an overload condition is an + implementation decision. + + When a reporting node that has selected the loss abatement algorithm + determines the need to request a reduction in traffic, it includes an + OC-OLR AVP in answer messages as described in Section 5.2.3. + + When sending the OC-OLR AVP, the reporting node MUST indicate a + percentage reduction in the OC-Reduction-Percentage AVP. + + The reporting node MAY change the reduction percentage in subsequent + overload reports. When doing so, the reporting node must conform to + overload report handling specified in Section 5.2.3. + +6.3. Reacting Node Behavior + + The method a reacting node uses to determine which request messages + are given abatement treatment is an implementation decision. + + When receiving an OC-OLR in an answer message where the algorithm + indicated in the OC-Supported-Features AVP is the loss algorithm, the + reacting node MUST apply abatement treatment to the requested + percentage of request messages sent. + + Note: The loss algorithm is a stateless algorithm. As a result, + the reacting node does not guarantee that there will be an + absolute reduction in traffic sent. Rather, it guarantees that + the requested percentage of new requests will be given abatement + treatment. + + If the reacting node comes out of the 100% traffic reduction + (meaning, it has received an OLR indicating that no traffic should be + sent, as a result of the overload report timing out), the reacting + node sending the traffic SHOULD be conservative and, for example, + first send "probe" messages to learn the overload condition of the + overloaded node before converging to any traffic amount/rate decided + by the sender. Similar concerns apply in all cases when the overload + report times out, unless the previous overload report stated 0% + reduction. + + + + + +Korhonen, et al. Standards Track [Page 24] + +RFC 7683 DOIC October 2015 + + + Note: The goal of this behavior is to reduce the probability of + overload condition thrashing where an immediate transition from + 100% reduction to 0% reduction results in the reporting node + moving quickly back into an overload condition. + +7. Attribute Value Pairs + + This section describes the encoding and semantics of the Diameter + Overload Indication Attribute Value Pairs (AVPs) defined in this + document. + + Refer to Section 4 of [RFC6733] for more information on AVPs and AVP + data types. + +7.1. OC-Supported-Features AVP + + The OC-Supported-Features AVP (AVP Code 621) is of type Grouped and + serves two purposes. First, it announces a node's support for the + DOIC solution in general. Second, it contains the description of the + supported DOIC features of the sending node. The OC-Supported- + Features AVP MUST be included in every Diameter request message a + DOIC supporting node sends. + + OC-Supported-Features ::= < AVP Header: 621 > + [ OC-Feature-Vector ] + * [ AVP ] + +7.2. OC-Feature-Vector AVP + + The OC-Feature-Vector AVP (AVP Code 622) is of type Unsigned64 and + contains a 64-bit flags field of announced capabilities of a DOIC + node. The value of zero (0) is reserved. + + The OC-Feature-Vector sub-AVP is used to announce the DOIC features + supported by the DOIC node, in the form of a flag-bits field in which + each bit announces one feature or capability supported by the node. + The absence of the OC-Feature-Vector AVP in request messages + indicates that only the default traffic abatement algorithm described + in this specification is supported. The absence of the OC-Feature- + Vector AVP in answer messages indicates that the default traffic + abatement algorithm described in this specification is selected + (while other traffic abatement algorithms may be supported), and no + features other than abatement algorithms are supported. + + + + + + + + +Korhonen, et al. Standards Track [Page 25] + +RFC 7683 DOIC October 2015 + + + The following capability is defined in this document: + + OLR_DEFAULT_ALGO (0x0000000000000001) + + When this flag is set by the a DOIC reacting node, it means that + the default traffic abatement (loss) algorithm is supported. When + this flag is set by a DOIC reporting node, it means that the loss + algorithm will be used for requested overload abatement. + +7.3. OC-OLR AVP + + The OC-OLR AVP (AVP Code 623) is of type Grouped and contains the + information necessary to convey an overload report on an overload + condition at the reporting node. The application the OC-OLR AVP + applies to is identified by the Application-ID found in the Diameter + message header. The host or realm the OC-OLR AVP concerns is + determined from the Origin-Host AVP and/or Origin-Realm AVP found in + the encapsulating Diameter command. The OC-OLR AVP is intended to be + sent only by a reporting node. + + OC-OLR ::= < AVP Header: 623 > + < OC-Sequence-Number > + < OC-Report-Type > + [ OC-Reduction-Percentage ] + [ OC-Validity-Duration ] + * [ AVP ] + +7.4. OC-Sequence-Number AVP + + The OC-Sequence-Number AVP (AVP Code 624) is of type Unsigned64. Its + usage in the context of overload control is described in Section 5.2. + + From the functionality point of view, the OC-Sequence-Number AVP is + used as a nonvolatile increasing counter for a sequence of overload + reports between two DOIC nodes for the same overload occurrence. + Sequence numbers are treated in a unidirectional manner, i.e., two + sequence numbers in each direction between two DOIC nodes are not + related or correlated. + +7.5. OC-Validity-Duration AVP + + The OC-Validity-Duration AVP (AVP Code 625) is of type Unsigned32 and + indicates in seconds the validity time of the overload report. The + number of seconds is measured after reception of the first OC-OLR AVP + with a given value of OC-Sequence-Number AVP. The default value for + the OC-Validity-Duration AVP is 30 seconds. When the OC-Validity- + Duration AVP is not present in the OC-OLR AVP, the default value + applies. The maximum value for the OC-Validity-Duration AVP is + + + +Korhonen, et al. Standards Track [Page 26] + +RFC 7683 DOIC October 2015 + + + 86,400 seconds (24 hours). If the value received in the OC-Validity- + Duration is greater than the maximum value, then the default value + applies. + +7.6. OC-Report-Type AVP + + The OC-Report-Type AVP (AVP Code 626) is of type Enumerated. The + value of the AVP describes what the overload report concerns. The + following values are initially defined: + + HOST_REPORT 0 + The overload report is for a host. Overload abatement treatment + applies to host-routed requests. + + REALM_REPORT 1 + The overload report is for a realm. Overload abatement treatment + applies to realm-routed requests. + + The values 2-4294967295 are unassigned. + +7.7. OC-Reduction-Percentage AVP + + The OC-Reduction-Percentage AVP (AVP Code 627) is of type Unsigned32 + and describes the percentage of the traffic that the sender is + requested to reduce, compared to what it otherwise would send. The + OC-Reduction-Percentage AVP applies to the default (loss) algorithm + specified in this specification. However, the AVP can be reused for + future abatement algorithms, if its semantics fit into the new + algorithm. + + The value of the Reduction-Percentage AVP is between zero (0) and one + hundred (100). Values greater than 100 are ignored. The value of + 100 means that all traffic is to be throttled, i.e., the reporting + node is under a severe load and ceases to process any new messages. + The value of 0 means that the reporting node is in a stable state and + has no need for the reacting node to apply any traffic abatement. + + + + + + + + + + + + + + + +Korhonen, et al. Standards Track [Page 27] + +RFC 7683 DOIC October 2015 + + +7.8. AVP Flag Rules + + +---------+ + |AVP flag | + |rules | + +----+----+ + AVP Section | |MUST| + Attribute Name Code Defined Value Type |MUST| NOT| + +--------------------------------------------------+----+----+ + |OC-Supported-Features 621 7.1 Grouped | | V | + +--------------------------------------------------+----+----+ + |OC-Feature-Vector 622 7.2 Unsigned64 | | V | + +--------------------------------------------------+----+----+ + |OC-OLR 623 7.3 Grouped | | V | + +--------------------------------------------------+----+----+ + |OC-Sequence-Number 624 7.4 Unsigned64 | | V | + +--------------------------------------------------+----+----+ + |OC-Validity-Duration 625 7.5 Unsigned32 | | V | + +--------------------------------------------------+----+----+ + |OC-Report-Type 626 7.6 Enumerated | | V | + +--------------------------------------------------+----+----+ + |OC-Reduction | | | + | -Percentage 627 7.7 Unsigned32 | | V | + +--------------------------------------------------+----+----+ + + As described in the Diameter base protocol [RFC6733], the M-bit usage + for a given AVP in a given command may be defined by the application. + +8. Error Response Codes + + When a DOIC node rejects a Diameter request due to overload, the DOIC + node MUST select an appropriate error response code. This + determination is made based on the probability of the request + succeeding if retried on a different path. + + Note: This only applies for DOIC nodes that are not the originator + of the request. + + A reporting node rejecting a Diameter request due to an overload + condition SHOULD send a DIAMETER_TOO_BUSY error response, if it can + assume that the same request may succeed on a different path. + + If a reporting node knows or assumes that the same request will not + succeed on a different path, the DIAMETER_UNABLE_TO_COMPLY error + response SHOULD be used. Retrying would consume valuable resources + during an occurrence of overload. + + + + + +Korhonen, et al. Standards Track [Page 28] + +RFC 7683 DOIC October 2015 + + + For instance, if the request arrived at the reporting node without + a Destination-Host AVP, then the reporting node might determine + that there is an alternative Diameter node that could successfully + process the request and that retrying the transaction would not + negatively impact the reporting node. DIAMETER_TOO_BUSY would be + sent in this case. + + If the request arrived at the reporting node with a Destination- + Host AVP populated with its own Diameter identity, then the + reporting node can assume that retrying the request would result + in it coming to the same reporting node. + DIAMETER_UNABLE_TO_COMPLY would be sent in this case. + + A second example is when an agent that supports the DOIC solution + is performing the role of a reacting node for a non-supporting + client. Requests that are rejected as a result of DOIC throttling + by the agent in this scenario would generally be rejected with a + DIAMETER_UNABLE_TO_COMPLY response code. + +9. IANA Considerations + +9.1. AVP Codes + + New AVPs defined by this specification are listed in Section 7. All + AVP codes are allocated from the "AVP Codes" sub-registry under the + "Authentication, Authorization, and Accounting (AAA) Parameters" + registry. + +9.2. New Registries + + Two new registries have been created in the "AVP Specific Values" + sub-registry under the "Authentication, Authorization, and Accounting + (AAA) Parameters" registry. + + A new "OC-Feature-Vector AVP Values (code 622)" registry has been + created. This registry contains the following: + + Feature Vector Value Name + + Feature Vector Value + + Specification defining the new value + + See Section 7.2 for the initial Feature Vector Value in the registry. + This specification defines the value. New values can be added to the + registry using the Specification Required policy [RFC5226]. + + + + + +Korhonen, et al. Standards Track [Page 29] + +RFC 7683 DOIC October 2015 + + + A new "OC-Report-Type AVP Values (code 626)" registry has been + created. This registry contains the following: + + Report Type Value Name + + Report Type Value + + Specification defining the new value + + See Section 7.6 for the initial assignment in the registry. New + types can be added using the Specification Required policy [RFC5226]. + +10. Security Considerations + + DOIC gives Diameter nodes the ability to request that downstream + nodes send fewer Diameter requests. Nodes do this by exchanging + overload reports that directly effect this reduction. This exchange + is potentially subject to multiple methods of attack and has the + potential to be used as a denial-of-service (DoS) attack vector. For + instance, a series of injected realm OLRs with a requested reduction + percentage of 100% could be used to completely eliminate any traffic + from being sent to that realm. + + Overload reports may contain information about the topology and + current status of a Diameter network. This information is + potentially sensitive. Network operators may wish to control + disclosure of overload reports to unauthorized parties to avoid their + use for competitive intelligence or to target attacks. + + Diameter does not include features to provide end-to-end + authentication, integrity protection, or confidentiality. This may + cause complications when sending overload reports between non- + adjacent nodes. + +10.1. Potential Threat Modes + + The Diameter protocol involves transactions in the form of requests + and answers exchanged between clients and servers. These clients and + servers may be peers, that is, they may share a direct transport + (e.g., TCP or SCTP) connection, or the messages may traverse one or + more intermediaries, known as Diameter Agents. Diameter nodes use + TLS, DTLS, or IPsec to authenticate peers and to provide + confidentiality and integrity protection of traffic between peers. + Nodes can make authorization decisions based on the peer identities + authenticated at the transport layer. + + + + + + +Korhonen, et al. Standards Track [Page 30] + +RFC 7683 DOIC October 2015 + + + When agents are involved, this presents an effectively transitive + trust model. That is, a Diameter client or server can authorize an + agent for certain actions, but it must trust that agent to make + appropriate authorization decisions about its peers, and so on. + Since confidentiality and integrity protection occur at the transport + layer, agents can read, and perhaps modify, any part of a Diameter + message, including an overload report. + + There are several ways an attacker might attempt to exploit the + overload control mechanism. An unauthorized third party might inject + an overload report into the network. If this third party is upstream + of an agent, and that agent fails to apply proper authorization + policies, downstream nodes may mistakenly trust the report. This + attack is at least partially mitigated by the assumption that nodes + include overload reports in Diameter answers but not in requests. + This requires an attacker to have knowledge of the original request + in order to construct an answer. Such an answer would also need to + arrive at a Diameter node via a protected transport connection. + Therefore, implementations MUST validate that an answer containing an + overload report is a properly constructed response to a pending + request prior to acting on the overload report, and that the answer + was received via an appropriate transport connection. + + A similar attack involves a compromised but otherwise authorized node + that sends an inappropriate overload report. For example, a server + for the realm "example.com" might send an overload report indicating + that a competitor's realm "example.net" is overloaded. If other + nodes act on the report, they may falsely believe that "example.net" + is overloaded, effectively reducing that realm's capacity. + Therefore, it's critical that nodes validate that an overload report + received from a peer actually falls within that peer's responsibility + before acting on the report or forwarding the report to other peers. + For example, an overload report from a peer that applies to a realm + not handled by that peer is suspect. This may require out-of-band, + non-Diameter agreements and/or mechanisms. + + This attack is partially mitigated by the fact that the + application, as well as host and realm, for a given OLR is + determined implicitly by respective AVPs in the enclosing answer. + If a reporting node modifies any of those AVPs, the enclosing + transaction will also be affected. + +10.2. Denial-of-Service Attacks + + Diameter overload reports, especially realm reports, can cause a node + to cease sending some or all Diameter requests for an extended + period. This makes them a tempting vector for DoS attacks. + Furthermore, since Diameter is almost always used in support of other + + + +Korhonen, et al. Standards Track [Page 31] + +RFC 7683 DOIC October 2015 + + + protocols, a DoS attack on Diameter is likely to impact those + protocols as well. In the worst case, where the Diameter application + is being used for access control into an IP network, a coordinated + DoS attack could result in the blockage of all traffic into that + network. Therefore, Diameter nodes MUST NOT honor or forward OLRs + received from peers that are not trusted to send them. + + An attacker might use the information in an OLR to assist in DoS + attacks. For example, an attacker could use information about + current overload conditions to time an attack for maximum effect, or + use subsequent overload reports as a feedback mechanism to learn the + results of a previous or ongoing attack. Operators need the ability + to ensure that OLRs are not leaked to untrusted parties. + +10.3. Noncompliant Nodes + + In the absence of an overload control mechanism, Diameter nodes need + to implement strategies to protect themselves from floods of + requests, and to make sure that a disproportionate load from one + source does not prevent other sources from receiving service. For + example, a Diameter server might throttle a certain percentage of + requests from sources that exceed certain limits. Overload control + can be thought of as an optimization for such strategies, where + downstream nodes never send the excess requests in the first place. + However, the presence of an overload control mechanism does not + remove the need for these other protection strategies. + + When a Diameter node sends an overload report, it cannot assume that + all nodes will comply, even if they indicate support for DOIC. A + noncompliant node might continue to send requests with no reduction + in load. Such noncompliance could be done accidentally or + maliciously to gain an unfair advantage over compliant nodes. + Requirement 28 in [RFC7068] indicates that the overload control + solution cannot assume that all Diameter nodes in a network are + trusted. It also requires that malicious nodes not be allowed to + take advantage of the overload control mechanism to get more than + their fair share of service. + +10.4. End-to-End Security Issues + + The lack of end-to-end integrity features makes it difficult to + establish trust in overload reports received from non-adjacent nodes. + Any agents in the message path may insert or modify overload reports. + Nodes must trust that their adjacent peers perform proper checks on + overload reports from their peers, and so on, creating a transitive- + trust requirement extending for potentially long chains of nodes. + Network operators must determine if this transitive trust requirement + is acceptable for their deployments. Nodes supporting Diameter + + + +Korhonen, et al. Standards Track [Page 32] + +RFC 7683 DOIC October 2015 + + + overload control MUST give operators the ability to select which + peers are trusted to deliver overload reports and whether they are + trusted to forward overload reports from non-adjacent nodes. DOIC + nodes MUST strip DOIC AVPs from messages received from peers that are + not trusted for DOIC purposes. + + The lack of end-to-end confidentiality protection means that any + Diameter Agent in the path of an overload report can view the + contents of that report. In addition to the requirement to select + which peers are trusted to send overload reports, operators MUST be + able to select which peers are authorized to receive reports. A node + MUST NOT send an overload report to a peer not authorized to receive + it. Furthermore, an agent MUST remove any overload reports that + might have been inserted by other nodes before forwarding a Diameter + message to a peer that is not authorized to receive overload reports. + + A DOIC node cannot always automatically detect that a peer also + supports DOIC. For example, a node might have a peer that is a + non-supporting agent. If nodes on the other side of that agent + send OC-Supported-Features AVPs, the agent is likely to forward + them as unknown AVPs. Messages received across the non-supporting + agent may be indistinguishable from messages received across a + DOIC supporting agent, giving the false impression that the non- + supporting agent actually supports DOIC. This complicates the + transitive-trust nature of DOIC. Operators need to be careful to + avoid situations where a non-supporting agent is mistakenly + trusted to enforce DOIC-related authorization policies. + + It is expected that work on end-to-end Diameter security might make + it easier to establish trust in non-adjacent nodes for overload + control purposes. Readers should be reminded, however, that the + overload control mechanism allows Diameter Agents to modify AVPs in, + or insert additional AVPs into, existing messages that are originated + by other nodes. If end-to-end security is enabled, there is a risk + that such modification could violate integrity protection. The + details of using any future Diameter end-to-end security mechanism + with overload control will require careful consideration, and are + beyond the scope of this document. + + + + + + + + + + + + + +Korhonen, et al. Standards Track [Page 33] + +RFC 7683 DOIC October 2015 + + +11. References + +11.1. Normative References + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, + DOI 10.17487/RFC2119, March 1997, + <http://www.rfc-editor.org/info/rfc2119>. + + [RFC5226] Narten, T. and H. Alvestrand, "Guidelines for Writing an + IANA Considerations Section in RFCs", BCP 26, RFC 5226, + DOI 10.17487/RFC5226, May 2008, + <http://www.rfc-editor.org/info/rfc5226>. + + [RFC6733] Fajardo, V., Ed., Arkko, J., Loughney, J., and G. Zorn, + Ed., "Diameter Base Protocol", RFC 6733, + DOI 10.17487/RFC6733, October 2012, + <http://www.rfc-editor.org/info/rfc6733>. + +11.2. Informative References + + [Cx] 3GPP, "Cx and Dx interfaces based on the Diameter + protocol; Protocol details", 3GPP TS 29.229 12.7.0, + September 2015. + + [PCC] 3GPP, "Policy and charging control architecture", 3GPP + TS 23.203 12.10.0, September 2015. + + [RFC4006] Hakala, H., Mattila, L., Koskinen, J-P., Stura, M., and J. + Loughney, "Diameter Credit-Control Application", RFC 4006, + DOI 10.17487/RFC4006, August 2005, + <http://www.rfc-editor.org/info/rfc4006>. + + [RFC7068] McMurry, E. and B. Campbell, "Diameter Overload Control + Requirements", RFC 7068, DOI 10.17487/RFC7068, November + 2013, <http://www.rfc-editor.org/info/rfc7068>. + + [S13] 3GPP, "Evolved Packet System (EPS); Mobility Management + Entity (MME) and Serving GPRS Support Node (SGSN) related + interfaces based on Diameter protocol", 3GPP TS 29.272 + 12.8.0, September 2015. + + + + + + + + + + +Korhonen, et al. Standards Track [Page 34] + +RFC 7683 DOIC October 2015 + + +Appendix A. Issues Left for Future Specifications + + The base solution for overload control does not cover all possible + use cases. A number of solution aspects were intentionally left for + future specification and protocol work. The following subsections + define some of the potential extensions to the DOIC solution. + +A.1. Additional Traffic Abatement Algorithms + + This specification describes only means for a simple loss-based + algorithm. Future algorithms can be added using the designed + solution extension mechanism. The new algorithms need to be + registered with IANA. See Sections 7.2 and 9 for the required IANA + steps. + +A.2. Agent Overload + + This specification focuses on Diameter endpoint (server or client) + overload. A separate extension will be required to outline the + handling of the case of agent overload. + +A.3. New Error Diagnostic AVP + + This specification indicates the use of existing error messages when + nodes reject requests due to overload. There is an expectation that + additional error codes or AVPs will be defined in a separate + specification to indicate that overload was the reason for the + rejection of the message. + +Appendix B. Deployment Considerations + + Non-supporting Agents + + Due to the way that realm-routed requests are handled in Diameter + networks with the server selection for the request done by an + agent, network operators should enable DOIC at agents that perform + server selection first. + + Topology-Hiding Interactions + + There exist proxies that implement what is referred to as Topology + Hiding. This can include cases where the agent modifies the + Origin-Host in answer messages. The behavior of the DOIC solution + is not well understood when this happens. As such, the DOIC + solution does not address this scenario. + + + + + + +Korhonen, et al. Standards Track [Page 35] + +RFC 7683 DOIC October 2015 + + + Inter-Realm/Administrative Domain Considerations + + There are likely to be special considerations for handling DOIC + signaling across administrative boundaries. This includes + considerations for whether or not information included in the DOIC + signaling should be sent across those boundaries. In addition, + consideration should be taken as to whether or not a reacting node + in one realm can be trusted to implement the requested overload + abatement handling for overload reports received from a separately + administered realm. + +Appendix C. Considerations for Applications Integrating the DOIC + Solution + + This section outlines considerations to be taken into account when + integrating the DOIC solution into Diameter applications. + +C.1. Application Classification + + The following is a classification of Diameter applications and + request types. This discussion is meant to document factors that + play into decisions made by the Diameter entity responsible for + handling overload reports. + + Section 8.1 of [RFC6733] defines two state machines that imply two + types of applications, session-less and session-based applications. + The primary difference between these types of applications is the + lifetime of Session-Ids. + + For session-based applications, the Session-Id is used to tie + multiple requests into a single session. + + The Credit-Control application defined in [RFC4006] is an example of + a Diameter session-based application. + + In session-less applications, the lifetime of the Session-Id is a + single Diameter transaction, i.e., the session is implicitly + terminated after a single Diameter transaction and a new Session-Id + is generated for each Diameter request. + + + + + + + + + + + + +Korhonen, et al. Standards Track [Page 36] + +RFC 7683 DOIC October 2015 + + + For the purposes of this discussion, session-less applications are + further divided into two types of applications: + + Stateless Applications: + + Requests within a stateless application have no relationship to + each other. The 3GPP-defined S13 application is an example of a + stateless application [S13], where only a Diameter command is + defined between a client and a server and no state is maintained + between two consecutive transactions. + + Pseudo-Session Applications: + + Applications that do not rely on the Session-Id AVP for + correlation of application messages related to the same session + but use other session-related information in the Diameter requests + for this purpose. The 3GPP-defined Cx application [Cx] is an + example of a pseudo-session application. + + The handling of overload reports must take the type of application + into consideration, as discussed in Appendix C.2. + +C.2. Implications of Application Type Overload + + This section discusses considerations for mitigating overload + reported by a Diameter entity. This discussion focuses on the type + of application. Appendix C.3 discusses considerations for handling + various request types when the target server is known to be in an + overloaded state. + + These discussions assume that the strategy for mitigating the + reported overload is to reduce the overall workload sent to the + overloaded entity. The concept of applying overload treatment to + requests targeted for an overloaded Diameter entity is inherent to + this discussion. The method used to reduce offered load is not + specified here, but it could include routing requests to another + Diameter entity known to be able to handle them, or it could mean + rejecting certain requests. For a Diameter Agent, rejecting requests + will usually mean generating appropriate Diameter error responses. + For a Diameter client, rejecting requests will depend upon the + application. For example, it could mean giving an indication to the + entity requesting the Diameter service that the network is busy and + to try again later. + + + + + + + + +Korhonen, et al. Standards Track [Page 37] + +RFC 7683 DOIC October 2015 + + + Stateless Applications: + + By definition, there is no relationship between individual + requests in a stateless application. As a result, when a request + is sent or relayed to an overloaded Diameter entity -- either a + Diameter Server or a Diameter Agent -- the sending or relaying + entity can choose to apply the overload treatment to any request + targeted for the overloaded entity. + + Pseudo-session Applications: + + For pseudo-session applications, there is an implied ordering of + requests. As a result, decisions about which requests towards an + overloaded entity to reject could take the command code of the + request into consideration. This generally means that + transactions later in the sequence of transactions should be given + more favorable treatment than messages earlier in the sequence. + This is because more work has already been done by the Diameter + network for those transactions that occur later in the sequence. + Rejecting them could result in increasing the load on the network + as the transactions earlier in the sequence might also need to be + repeated. + + Session-Based Applications: + + Overload handling for session-based applications must take into + consideration the work load associated with setting up and + maintaining a session. As such, the entity sending requests + towards an overloaded Diameter entity for a session-based + application might tend to reject new session requests prior to + rejecting intra-session requests. In addition, session-ending + requests might be given a lower probability of being rejected, as + rejecting session-ending requests could result in session status + being out of sync between the Diameter clients and servers. + Application designers that would decide to reject mid-session + requests will need to consider whether the rejection invalidates + the session and any resulting session cleanup procedures. + +C.3. Request Transaction Classification + + Independent Request: + + An independent request is not correlated to any other requests, + and, as such, the lifetime of the Session-Id is constrained to an + individual transaction. + + + + + + +Korhonen, et al. Standards Track [Page 38] + +RFC 7683 DOIC October 2015 + + + Session-Initiating Request: + + A session-initiating request is the initial message that + establishes a Diameter session. The ACR message defined in + [RFC6733] is an example of a session-initiating request. + + Correlated Session-Initiating Request: + + There are cases when multiple session-initiated requests must be + correlated and managed by the same Diameter server. It is notably + the case in the 3GPP Policy and Charging Control (PCC) + architecture [PCC], where multiple apparently independent Diameter + application sessions are actually correlated and must be handled + by the same Diameter server. + + Intra-session Request: + + An intra-session request is a request that uses the same Session- + Id as the one used in a previous request. An intra-session + request generally needs to be delivered to the server that handled + the session-creating request for the session. The STR message + defined in [RFC6733] is an example of an intra-session request. + + Pseudo-session Requests: + + Pseudo-session requests are independent requests and do not use + the same Session-Id but are correlated by other session-related + information contained in the request. There exist Diameter + applications that define an expected ordering of transactions. + This sequencing of independent transactions results in a pseudo- + session. The AIR, MAR, and SAR requests in the 3GPP-defined Cx + [Cx] application are examples of pseudo-session requests. + +C.4. Request Type Overload Implications + + The request classes identified in Appendix C.3 have implications on + decisions about which requests should be throttled first. The + following list of request treatments regarding throttling is provided + as guidelines for application designers when implementing the + Diameter overload control mechanism described in this document. The + exact behavior regarding throttling is a matter of local policy, + unless specifically defined for the application. + + Independent Requests: + + Independent requests can generally be given equal treatment when + making throttling decisions, unless otherwise indicated by + application requirements or local policy. + + + +Korhonen, et al. Standards Track [Page 39] + +RFC 7683 DOIC October 2015 + + + Session-Initiating Requests: + + Session-initiating requests often represent more work than + independent or intra-session requests. Moreover, session- + initiating requests are typically followed by other session- + related requests. Since the main objective of overload control is + to reduce the total number of requests sent to the overloaded + entity, throttling decisions might favor allowing intra-session + requests over session-initiating requests. In the absence of + local policies or application-specific requirements to the + contrary, individual session-initiating requests can be given + equal treatment when making throttling decisions. + + Correlated Session-Initiating Requests: + + A request that results in a new binding; where the binding is used + for routing of subsequent session-initiating requests to the same + server, it represents more work load than other requests. As + such, these requests might be throttled more frequently than other + request types. + + Pseudo-session Requests: + + Throttling decisions for pseudo-session requests can take into + consideration where individual requests fit into the overall + sequence of requests within the pseudo-session. Requests that are + earlier in the sequence might be throttled more aggressively than + requests that occur later in the sequence. + + Intra-session Requests: + + There are two types of intra-sessions requests, requests that + terminate a session and the remainder of intra-session requests. + Implementers and operators may choose to throttle session- + terminating requests less aggressively in order to gracefully + terminate sessions, allow cleanup of the related resources (e.g., + session state), and avoid the need for additional intra-session + requests. Favoring session termination requests may reduce the + session management impact on the overloaded entity. The default + handling of other intra-session requests might be to treat them + equally when making throttling decisions. There might also be + application-level considerations whether some request types are + favored over others. + + + + + + + + +Korhonen, et al. Standards Track [Page 40] + +RFC 7683 DOIC October 2015 + + +Contributors + + The following people contributed substantial ideas, feedback, and + discussion to this document: + + o Eric McMurry + + o Hannes Tschofenig + + o Ulrich Wiehe + + o Jean-Jacques Trottin + + o Maria Cruz Bartolome + + o Martin Dolly + + o Nirav Salot + + o Susan Shishufeng + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Korhonen, et al. Standards Track [Page 41] + +RFC 7683 DOIC October 2015 + + +Authors' Addresses + + Jouni Korhonen (editor) + Broadcom Corporation + 3151 Zanker Road + San Jose, CA 95134 + United States + + Email: [email protected] + + + Steve Donovan (editor) + Oracle + 7460 Warren Parkway + Frisco, Texas 75034 + United States + + Email: [email protected] + + + Ben Campbell + Oracle + 7460 Warren Parkway + Frisco, Texas 75034 + United States + + Email: [email protected] + + + Lionel Morand + Orange Labs + 38/40 rue du General Leclerc + Issy-Les-Moulineaux Cedex 9 92794 + France + + Phone: +33145296257 + Email: [email protected] + + + + + + + + + + + + + + +Korhonen, et al. Standards Track [Page 42] + diff --git a/lib/diameter/examples/code/client.erl b/lib/diameter/examples/code/client.erl index 6fb90b1c09..0864919cdd 100644 --- a/lib/diameter/examples/code/client.erl +++ b/lib/diameter/examples/code/client.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2015. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -39,7 +39,6 @@ -module(client). -include_lib("diameter/include/diameter.hrl"). --include_lib("diameter/include/diameter_gen_base_rfc6733.hrl"). -export([start/1, %% start a service start/2, %% @@ -71,6 +70,7 @@ {'Product-Name', "Client"}, {'Auth-Application-Id', [0]}, {string_decode, false}, + {decode_format, map}, {application, [{alias, common}, {dictionary, diameter_gen_base_rfc6733}, {module, client_cb}]}]). @@ -108,9 +108,9 @@ connect(T) -> call(Name) -> SId = diameter:session_id(?L(Name)), - RAR = #diameter_base_RAR{'Session-Id' = SId, - 'Auth-Application-Id' = 0, - 'Re-Auth-Request-Type' = 0}, + RAR = ['RAR' | #{'Session-Id' => SId, + 'Auth-Application-Id' => 0, + 'Re-Auth-Request-Type' => 0}], diameter:call(Name, common, RAR, []). call() -> diff --git a/lib/diameter/examples/code/client_cb.erl b/lib/diameter/examples/code/client_cb.erl index ed1d3b9b7b..af2d4d6da7 100644 --- a/lib/diameter/examples/code/client_cb.erl +++ b/lib/diameter/examples/code/client_cb.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -55,21 +55,18 @@ prepare_request(#diameter_packet{msg = ['RAR' = T | Avps]}, _, {_, Caps}) -> origin_realm = {OR, DR}} = Caps, - {send, [T, {'Origin-Host', OH}, - {'Origin-Realm', OR}, - {'Destination-Host', DH}, - {'Destination-Realm', DR} - | Avps]}; - -prepare_request(#diameter_packet{msg = Rec}, _, {_, Caps}) -> - #diameter_caps{origin_host = {OH, DH}, - origin_realm = {OR, DR}} - = Caps, - - {send, Rec#diameter_base_RAR{'Origin-Host' = OH, - 'Origin-Realm' = OR, - 'Destination-Host' = DH, - 'Destination-Realm' = DR}}. + {send, [T | if is_map(Avps) -> + Avps#{'Origin-Host' => OH, + 'Origin-Realm' => OR, + 'Destination-Host' => DH, + 'Destination-Realm' => DR}; + is_list(Avps) -> + [{'Origin-Host', OH}, + {'Origin-Realm', OR}, + {'Destination-Host', DH}, + {'Destination-Realm', DR} + | Avps] + end]}. %% prepare_retransmit/3 diff --git a/lib/diameter/examples/code/node.erl b/lib/diameter/examples/code/node.erl index 246be4194b..fc5830f8e2 100644 --- a/lib/diameter/examples/code/node.erl +++ b/lib/diameter/examples/code/node.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2015. All Rights Reserved. +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,6 +30,8 @@ connect/2, stop/1]). +-export([message/3]). + -type protocol() :: tcp | sctp. @@ -128,6 +130,8 @@ stop(Name) -> server_opts({T, Addr, Port}) -> [{transport_module, tmod(T)}, {transport_config, [{reuseaddr, true}, + {sender, true}, + {message_cb, [fun ?MODULE:message/3, 0]}, {ip, addr(Addr)}, {port, Port}]}]; @@ -173,3 +177,26 @@ addr(loopback) -> {127,0,0,1}; addr(A) -> A. + +%% --------------------------------------------------------------------------- + +%% message/3 +%% +%% Simple message callback that limits the number of concurrent +%% requests on the peer connection in question. + +%% Incoming request. +message(recv, <<_:32, 1:1, _/bits>> = Bin, N) -> + [Bin, N < 32, fun ?MODULE:message/3, N+1]; + +%% Outgoing request. +message(ack, <<_:32, 1:1, _/bits>>, _) -> + []; + +%% Incoming answer or request discarded. +message(ack, _, N) -> + [N =< 32, fun ?MODULE:message/3, N-1]; + +%% Outgoing message or incoming answer. +message(_, Bin, _) -> + [Bin]. diff --git a/lib/diameter/include/diameter_gen.hrl b/lib/diameter/include/diameter_gen.hrl index fb6370fe54..548763ec7d 100644 --- a/lib/diameter/include/diameter_gen.hrl +++ b/lib/diameter/include/diameter_gen.hrl @@ -26,13 +26,13 @@ %% encode_avps/3 -encode_avps(Name, Vals, Opts) -> - diameter_gen:encode_avps(Name, Vals, Opts#{module => ?MODULE}). +encode_avps(Name, Avps, Opts) -> + diameter_gen:encode_avps(Name, Avps, Opts#{module => ?MODULE}). %% decode_avps/2 -decode_avps(Name, Recs, Opts) -> - diameter_gen:decode_avps(Name, Recs, Opts#{module => ?MODULE}). +decode_avps(Name, Avps, Opts) -> + diameter_gen:decode_avps(Name, Avps, Opts#{module => ?MODULE}). %% avp/5 diff --git a/lib/diameter/src/Makefile b/lib/diameter/src/Makefile index 6bf748a727..3af856f63e 100644 --- a/lib/diameter/src/Makefile +++ b/lib/diameter/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2016. All Rights Reserved. +# Copyright Ericsson AB 2010-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -274,9 +274,7 @@ gen/diameter_gen_base_accounting.erl gen/diameter_gen_base_accounting.hrl: \ gen/diameter_gen_acct_rfc6733.erl gen/diameter_gen_acct_rfc6733.hrl: \ $(EBIN)/diameter_gen_base_rfc6733.$(EMULATOR) -gen/diameter_gen_relay.erl gen/diameter_gen_relay.hrl \ -gen/diameter_gen_base_rfc3588.erl gen/diameter_gen_base_rfc3588.hrl \ -gen/diameter_gen_base_rfc6733.erl gen/diameter_gen_base_rfc6733.hrl: \ +$(DICT_ERLS) $(DICT_HRLS): \ $(COMPILER_MODULES:%=$(EBIN)/%.$(EMULATOR)) $(DICT_MODULES:gen/%=$(EBIN)/%.$(EMULATOR)): \ diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index bd92e16fba..b90b794611 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -46,7 +46,10 @@ -export([start/0, stop/0]). --export_type([evaluable/0, +-export_type([eval/0, + evaluable/0, %% deprecated + decode_format/0, + strict_arities/0, restriction/0, message_length/0, remotes/0, @@ -299,7 +302,7 @@ call(SvcName, App, Message) -> | realm | {host, any|'DiameterIdentity'()} | {realm, any|'DiameterIdentity'()} - | {eval, evaluable()} + | {eval, eval()} | {neg, peer_filter()} | {all, [peer_filter()]} | {any, [peer_filter()]}. @@ -307,10 +310,13 @@ call(SvcName, App, Message) -> -opaque peer_ref() :: pid(). --type evaluable() +-type eval() :: {module(), atom(), list()} | fun() - | maybe_improper_list(evaluable(), list()). + | maybe_improper_list(eval(), list()). + +-type evaluable() + :: eval(). -type sequence() :: {'Unsigned32'(), 0..32}. @@ -320,29 +326,61 @@ call(SvcName, App, Message) -> | node | nodes | [node()] - | evaluable(). + | eval(). -type remotes() :: boolean() | [node()] - | evaluable(). + | eval(). -type message_length() :: 0..16#FFFFFF. +-type decode_format() + :: record + | list + | map + | none + | record_from_map. + +-type strict_arities() + :: false + | encode + | decode. + +%% Options common to both start_service/2 and add_transport/2. + +-type common_opt() + :: {pool_size, pos_integer()} + | {capabilities_cb, eval()} + | {capx_timeout, 'Unsigned32'()} + | {strict_capx, boolean()} + | {strict_mbit, boolean()} + | {avp_dictionaries, [module()]} + | {disconnect_cb, eval()} + | {dpr_timeout, 'Unsigned32'()} + | {dpa_timeout, 'Unsigned32'()} + | {incoming_maxlen, message_length()} + | {length_errors, exit | handle | discard} + | {connect_timer, 'Unsigned32'()} + | {watchdog_timer, 'Unsigned32'() | {module(), atom(), list()}} + | {watchdog_config, [{okay|suspect, non_neg_integer()}]} + | {spawn_opt, list()}. + %% Options passed to start_service/2 -type service_opt() :: capability() | {application, [application_opt()]} | {restrict_connections, restriction()} - | {sequence, sequence() | evaluable()} + | {sequence, sequence() | eval()} | {share_peers, remotes()} + | {decode_format, decode_format()} + | {traffic_counters, boolean()} | {string_decode, boolean()} - | {strict_mbit, boolean()} - | {incoming_maxlen, message_length()} + | {strict_arities, true | strict_arities()} | {use_shared_peers, remotes()} - | {spawn_opt, list()}. + | common_opt(). -type application_opt() :: {alias, app_alias()} @@ -372,20 +410,9 @@ call(SvcName, App, Message) -> :: {transport_module, atom()} | {transport_config, any()} | {transport_config, any(), 'Unsigned32'() | infinity} - | {pool_size, pos_integer()} | {applications, [app_alias()]} | {capabilities, [capability()]} - | {capabilities_cb, evaluable()} - | {capx_timeout, 'Unsigned32'()} - | {capx_strictness, boolean()} - | {disconnect_cb, evaluable()} - | {dpr_timeout, 'Unsigned32'()} - | {dpa_timeout, 'Unsigned32'()} - | {length_errors, exit | handle | discard} - | {connect_timer, 'Unsigned32'()} - | {watchdog_timer, 'Unsigned32'() | {module(), atom(), list()}} - | {watchdog_config, [{okay|suspect, non_neg_integer()}]} - | {spawn_opt, list()} + | common_opt() | {private, any()}. %% Predicate passed to remove_transport/2 diff --git a/lib/diameter/src/base/diameter_callback.erl b/lib/diameter/src/base/diameter_callback.erl index f9cdc66c70..d04a416bef 100644 --- a/lib/diameter/src/base/diameter_callback.erl +++ b/lib/diameter/src/base/diameter_callback.erl @@ -26,16 +26,16 @@ %% as the Diameter application callback in question. The record has %% one field for each callback function as well as 'default' and %% 'extra' fields. A function-specific field can be set to a -%% diameter:evaluable() in order to redirect the callback +%% diameter:eval() in order to redirect the callback %% corresponding to that field, or to 'false' to request the default %% callback implemented in this module. If neither of these fields are %% set then the 'default' field determines the form of the callback: a %% module name results in the usual callback as if the module had been -%% configured directly as the callback module, a diameter_evaluable() +%% configured directly as the callback module, a diameter_eval() %% in a callback applied to the atom-valued callback name and argument %% list. For all callbacks not to this module, the 'extra' field is a %% list of additional arguments, following arguments supplied by -%% diameter but preceding those of the diameter:evaluable() being +%% diameter but preceding those of the diameter:eval() being %% applied. %% %% For example, the following config to diameter:start_service/2, in diff --git a/lib/diameter/src/base/diameter_codec.erl b/lib/diameter/src/base/diameter_codec.erl index 82fa796e69..2dd2c906a2 100644 --- a/lib/diameter/src/base/diameter_codec.erl +++ b/lib/diameter/src/base/diameter_codec.erl @@ -29,7 +29,7 @@ msg_name/2, msg_id/1]). -%% Towards generated encoders (from diameter_gen.hrl). +%% towards diameter_gen -export([pack_data/2, pack_avp/2]). @@ -110,7 +110,7 @@ encode(Mod, Opts, Msg) -> enc(_, Opts, #diameter_packet{msg = [#diameter_header{} = Hdr | As]} = Pkt) -> - try encode_avps(reorder(As), Opts) of + try encode_avps(As, Opts) of Avps -> Bin = list_to_binary(Avps), Len = 20 + size(Bin), @@ -206,51 +206,12 @@ values(Avps) -> %% Message as a list of #diameter_avp{} ... encode_avps(_, _, [#diameter_avp{} | _] = Avps, Opts) -> - encode_avps(reorder(Avps), Opts); + encode_avps(Avps, Opts); %% ... or as a tuple list or record. encode_avps(Mod, MsgName, Values, Opts) -> Mod:encode_avps(MsgName, Values, Opts). -%% reorder/1 -%% -%% Reorder AVPs for the relay case using the index field of -%% diameter_avp records. Decode populates this field in collect_avps -%% and presents AVPs in reverse order. A relay then sends the reversed -%% list with a Route-Record AVP prepended. The goal here is just to do -%% lists:reverse/1 in Grouped AVPs and the outer list, but only in the -%% case there are indexed AVPs at all, so as not to reverse lists that -%% have been explicilty sent (unindexed, in the desired order) as a -%% diameter_avp list. The effect is the same as lists:keysort/2, but -%% only on the cases we expect, not a general sort. - -reorder(Avps) -> - case reorder(Avps, []) of - false -> - Avps; - Sorted -> - Sorted - end. - -%% reorder/3 - -%% In case someone has reversed the list already. (Not likely.) -reorder([#diameter_avp{index = 0} | _] = Avps, Acc) -> - Avps ++ Acc; - -%% Assume indexed AVPs are in reverse order. -reorder([#diameter_avp{index = N} = A | Avps], Acc) - when is_integer(N) -> - lists:reverse(Avps, [A | Acc]); - -%% An unindexed AVP. -reorder([H | T], Acc) -> - reorder(T, [H | Acc]); - -%% No indexed members. -reorder([], _) -> - false. - %% encode_avps/2 encode_avps(Avps, Opts) -> @@ -287,7 +248,8 @@ rec2msg(Mod, Rec) -> %% longer *the* decode. decode(Mod, Pkt) -> - Opts = #{string_decode => true, + Opts = #{decode_format => record, + string_decode => true, strict_mbit => true, rfc => 6733}, decode(Mod, Opts, Pkt). @@ -326,13 +288,7 @@ decode(Mod, AppMod, Opts, Pkt) -> %% Relay application: just extract the avp's without any decoding of %% their data since we don't know the application in question. decode(?APP_ID_RELAY, _, _, _, #diameter_packet{} = Pkt) -> - case collect_avps(Pkt) of - {E, As} -> - Pkt#diameter_packet{avps = As, - errors = [E]}; - As -> - Pkt#diameter_packet{avps = As} - end; + collect_avps(Pkt); %% Otherwise decode using the dictionary. decode(_, Mod, AppMod, Opts, #diameter_packet{header = Hdr} = Pkt) -> @@ -341,44 +297,50 @@ decode(_, Mod, AppMod, Opts, #diameter_packet{header = Hdr} = Pkt) -> is_error = IsError} = Hdr, - MsgName = if IsError andalso not IsRequest -> + MsgName = if IsError, not IsRequest -> 'answer-message'; true -> Mod:msg_name(CmdCode, IsRequest) end, - decode_avps(MsgName, Mod, AppMod, Opts, Pkt, collect_avps(Pkt)); + decode_avps(MsgName, Mod, AppMod, Opts, Pkt); decode(Id, Mod, AppMod, Opts, Bin) when is_binary(Bin) -> decode(Id, Mod, AppMod, Opts, #diameter_packet{header = decode_header(Bin), bin = Bin}). -%% decode_avps/6 - -decode_avps(MsgName, Mod, AppMod, Opts, Pkt, {E, Avps}) -> - ?LOG(invalid_avp_length, Pkt#diameter_packet.header), - #diameter_packet{errors = Failed} - = P - = decode_avps(MsgName, Mod, AppMod, Opts, Pkt, Avps), - P#diameter_packet{errors = [E | Failed]}; +%% decode_avps/5 -decode_avps('', _, _, _, Pkt, Avps) -> %% unknown message ... - ?LOG(unknown_message, Pkt#diameter_packet.header), - Pkt#diameter_packet{avps = lists:reverse(Avps), - errors = [3001]}; %% DIAMETER_COMMAND_UNSUPPORTED +decode_avps('', _, _, _, #diameter_packet{header = H, %% unknown message + bin = Bin} + = Pkt) -> + ?LOG(unknown_message, H), + Pkt#diameter_packet{avps = collect_avps(Bin), + errors = [3001]}; %% DIAMETER_COMMAND_UNSUPPORTED %% msg = undefined identifies this case. -decode_avps(MsgName, Mod, AppMod, Opts, Pkt, Avps) -> %% ... or not +decode_avps(MsgName, Mod, AppMod, Opts, #diameter_packet{bin = Bin} = Pkt) -> + {_, Avps} = split_binary(Bin, 20), {Rec, As, Errors} = Mod:decode_avps(MsgName, Avps, - Opts#{dictionary => AppMod, + Opts#{app_dictionary => AppMod, failed_avp => false}), ?LOGC([] /= Errors, decode_errors, Pkt#diameter_packet.header), - Pkt#diameter_packet{msg = Rec, + Pkt#diameter_packet{msg = reformat(MsgName, Rec, Opts), errors = Errors, avps = As}. +%% reformat/3 + +reformat(MsgName, Avps, #{decode_format := T}) + when T == map; + T == list -> + [MsgName | Avps]; + +reformat(_, Msg, _) -> + Msg. + %%% --------------------------------------------------------------------------- %%% # decode_header/1 %%% --------------------------------------------------------------------------- @@ -515,24 +477,21 @@ msg_id(<<_:32, Rbit:1, _:7, CmdCode:24, ApplId:32, _/binary>>) -> %%% # collect_avps/1 %%% --------------------------------------------------------------------------- -%% Note that the returned list of AVP's is reversed relative to their -%% order in the binary. Note also that grouped avp's aren't unraveled, -%% only those at the top level. +%% This is only used for the relay decode. Note that grouped avp's +%% aren't unraveled, only those at the top level. --spec collect_avps(#diameter_packet{} | binary()) - -> [Avp] - | {Error, [Avp]} - when Avp :: #diameter_avp{}, - Error :: {5014, #diameter_avp{}}. +-spec collect_avps(#diameter_packet{}) + -> #diameter_packet{}; + (binary()) + -> [#diameter_avp{}]. -collect_avps(#diameter_packet{bin = <<_:20/binary, Avps/binary>>}) -> - collect_avps(Avps, 0, []); +collect_avps(#diameter_packet{bin = Bin} = Pkt) -> + Pkt#diameter_packet{avps = collect_avps(Bin)}; -collect_avps(Bin) - when is_binary(Bin) -> - collect_avps(Bin, 0, []). +collect_avps(<<_:20/binary, Avps/binary>>) -> + collect(Avps). -%% collect_avps/3 +%% collect/1 %% 0 1 2 3 %% 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -546,66 +505,48 @@ collect_avps(Bin) %% | Data ... %% +-+-+-+-+-+-+-+-+ -collect_avps(<<Code:32, V:1, M:1, P:1, _:5, Len:24, I:V/unit:32, Rest/binary>>, - N, - Acc) -> - collect_avps(Code, - if 1 == V -> I; 0 == V -> undefined end, - 1 == M, - 1 == P, - Len - 8 - V*4, %% Might be negative, which ensures - ?PAD(Len), %% failure of the Data match below. - Rest, - N, - Acc); - -collect_avps(<<>>, _, Acc) -> - Acc; +collect(<<Code:32, V:1, M:1, P:1, _:5, Len:24, I:V/unit:32, Rest/binary>>) -> + collect(Rest, + Code, + if 1 == V -> I; 0 == V -> undefined end, + Len - 8 - V*4, %% Might be negative, which ensures + ?PAD(Len), %% failure of the match below. + 1 == M, + 1 == P); + +collect(<<>>) -> + []; %% Header is truncated. pack_avp/1 will pad this at encode if sent in %% a Failed-AVP. -collect_avps(Bin, _, Acc) -> - {{5014, #diameter_avp{data = Bin}}, Acc}. +collect(Bin) -> + [#diameter_avp{data = {5014, Bin}}]. -%% collect_avps/9 +%% collect/7 -%% Duplicate the diameter_avp creation in each branch below to avoid -%% modifying the record, which profiling has shown to be a relatively -%% costly part of building the list. - -collect_avps(Code, VendorId, M, P, Len, Pad, Rest, N, Acc) -> - case Rest of - <<Data:Len/binary, _:Pad/binary, T/binary>> -> +collect(Bin, Code, Vid, DataLen, Pad, M, P) -> + case Bin of + <<Data:DataLen/binary, _:Pad/binary, Rest/binary>> -> Avp = #diameter_avp{code = Code, - vendor_id = VendorId, + vendor_id = Vid, is_mandatory = M, need_encryption = P, - data = Data, - index = N}, - collect_avps(T, N+1, [Avp | Acc]); + data = Data}, + [Avp | collect(Rest)]; _ -> %% Length in header points past the end of the message, or - %% doesn't span the header. As stated in the 6733 text - %% above, it's sufficient to return a zero-filled minimal - %% payload if this is a request. Do this (in cases that we - %% know the type) by inducing a decode failure and letting - %% the dictionary's decode (in diameter_gen) deal with it. - %% - %% Note that the extra bit can only occur in the trailing - %% AVP of a message or Grouped AVP, since a faulty AVP - %% Length is otherwise indistinguishable from a correct - %% one here, as we don't know the types of the AVPs being - %% extracted. - Avp = #diameter_avp{code = Code, - vendor_id = VendorId, - is_mandatory = M, - need_encryption = P, - data = {5014, Rest}, - index = N}, - [Avp | Acc] + %% doesn't span the header. Note that an length error can + %% only occur in the trailing AVP of a message or Grouped + %% AVP, since a faulty AVP Length is otherwise + %% indistinguishable from a correct one here, as we don't + %% know the types of the AVPs being extracted. + [#diameter_avp{code = Code, + vendor_id = Vid, + is_mandatory = M, + need_encryption = P, + data = {5014, Bin}}] end. - %% 3588: %% %% DIAMETER_INVALID_AVP_LENGTH 5014 @@ -673,8 +614,8 @@ pack_avp(#diameter_avp{data = {T, {Type, Value}}}, Opts) -> pack_avp(#diameter_avp{data = {T, Data}}, _) -> pack_data(T, Data); -pack_avp(#diameter_avp{data = {Dict, Name, Data}}, Opts) -> - pack_data(Dict:avp_header(Name), Dict:avp(encode, Data, Name, Opts)); +pack_avp(#diameter_avp{data = {Dict, Name, Value}}, Opts) -> + pack_data(Dict:avp_header(Name), Dict:avp(encode, Value, Name, Opts)); %% ... with a truncated header ... pack_avp(#diameter_avp{code = undefined, data = B}, _) diff --git a/lib/diameter/src/base/diameter_config.erl b/lib/diameter/src/base/diameter_config.erl index 34018ae6d3..90a9282349 100644 --- a/lib/diameter/src/base/diameter_config.erl +++ b/lib/diameter/src/base/diameter_config.erl @@ -102,9 +102,6 @@ -record(monitor, {mref = make_ref() :: reference(), service}). %% name -%% The default sequence mask. --define(NOMASK, {0,32}). - %% Time to lay low before restarting a dead service. -define(RESTART_SLEEP, 2000). @@ -560,87 +557,186 @@ add(SvcName, Type, Opts0) -> end. transport_opts(Opts) -> - lists:map(fun topt/1, Opts). + [setopt(transport, T) || T <- Opts]. + +%% setopt/2 -topt(T) -> - case opt(T) of +setopt(K, T) -> + case opt(K, T) of {value, X} -> X; true -> T; false -> - ?THROW({invalid, T}) + ?THROW({invalid, T}); + {error, Reason} -> + ?THROW({invalid, T, Reason}) end. -opt({transport_module, M}) -> +%% opt/2 + +opt(_, {incoming_maxlen, N}) -> + is_integer(N) andalso 0 =< N andalso N < 1 bsl 24; + +opt(service, {K, B}) + when K == string_decode; + K == traffic_counters -> + is_boolean(B); + +opt(service, {K, false}) + when K == share_peers; + K == use_shared_peers; + K == monitor; + K == restrict_connections; + K == strict_arities -> + true; + +opt(service, {K, true}) + when K == share_peers; + K == use_shared_peers; + K == strict_arities -> + true; + +opt(service, {decode_format, T}) + when T == record; + T == list; + T == map; + T == none; + T == record_from_map -> + true; + +opt(service, {strict_arities, T}) + when T == encode; + T == decode -> + true; + +opt(service, {restrict_connections, T}) + when T == node; + T == nodes -> + true; + +opt(service, {K, T}) + when (K == share_peers + orelse K == use_shared_peers + orelse K == restrict_connections), ([] == T + orelse is_atom(hd(T))) -> + true; + +opt(service, {monitor, P}) -> + is_pid(P); + +opt(service, {K, F}) + when K == restrict_connections; + K == share_peers; + K == use_shared_peers -> + try diameter_lib:eval(F) of %% but no guarantee that it won't fail later + Nodes -> + is_list(Nodes) orelse {error, Nodes} + catch + E:R -> + {error, {E, R, ?STACK}} + end; + +opt(service, {sequence, {H,N}}) -> + 0 =< N andalso N =< 32 + andalso is_integer(H) + andalso 0 =< H + andalso 0 == H bsr (32-N); + +opt(service = S, {sequence = K, F}) -> + try diameter_lib:eval(F) of + {_,_} = T -> + KT = {K,T}, + opt(S, KT) andalso {value, KT}; + V -> + {error, V} + catch + E:R -> + {error, {E, R, ?STACK}} + end; + +opt(transport, {transport_module, M}) -> is_atom(M); -opt({transport_config, _, Tmo}) -> +opt(transport, {transport_config, _, Tmo}) -> ?IS_UINT32(Tmo) orelse Tmo == infinity; -opt({applications, As}) -> +opt(transport, {applications, As}) -> is_list(As); -opt({capabilities, Os}) -> - is_list(Os) andalso ok == encode_CER(Os); +opt(transport, {capabilities, Os}) -> + is_list(Os) andalso try ok = encode_CER(Os), true + catch ?FAILURE(No) -> {error, No} + end; -opt({K, Tmo}) +opt(_, {K, Tmo}) when K == capx_timeout; K == dpr_timeout; K == dpa_timeout -> ?IS_UINT32(Tmo); -opt({capx_strictness, B}) -> +opt(_, {capx_strictness, B}) -> + is_boolean(B) andalso {value, {strict_capx, B}}; +opt(_, {K, B}) + when K == strict_capx; + K == strict_mbit -> is_boolean(B); -opt({length_errors, T}) -> +opt(_, {avp_dictionaries, Mods}) -> + is_list(Mods) andalso lists:all(fun erlang:is_atom/1, Mods); + +opt(_, {length_errors, T}) -> lists:member(T, [exit, handle, discard]); -opt({K, Tmo}) - when K == reconnect_timer; %% deprecated - K == connect_timer -> +opt(transport, {reconnect_timer, Tmo}) -> %% deprecated + ?IS_UINT32(Tmo) andalso {value, {connect_timer, Tmo}}; +opt(_, {connect_timer, Tmo}) -> ?IS_UINT32(Tmo); -opt({watchdog_timer, {M,F,A}}) +opt(_, {watchdog_timer, {M,F,A}}) when is_atom(M), is_atom(F), is_list(A) -> true; -opt({watchdog_timer, Tmo}) -> +opt(_, {watchdog_timer, Tmo}) -> ?IS_UINT32(Tmo); -opt({watchdog_config, L}) -> - is_list(L) andalso lists:all(fun wdopt/1, L); +opt(_, {watchdog_config, L}) -> + is_list(L) andalso lists:all(fun wd/1, L); -opt({spawn_opt, {M,F,A}}) +opt(_, {spawn_opt, {M,F,A}}) when is_atom(M), is_atom(F), is_list(A) -> true; -opt({spawn_opt = K, Opts}) -> +opt(_, {spawn_opt = K, Opts}) -> if is_list(Opts) -> {value, {K, spawn_opts(Opts)}}; true -> false end; -opt({pool_size, N}) -> +opt(_, {pool_size, N}) -> is_integer(N) andalso 0 < N; -%% Options that we can't validate. -opt({K, _}) +%% Options we can't validate. +opt(_, {K, _}) + when K == disconnect_cb; + K == capabilities_cb -> + true; +opt(transport, {K, _}) when K == transport_config; - K == capabilities_cb; - K == disconnect_cb; K == private -> true; -%% Anything else, which is ignored by us. This makes options sensitive -%% to spelling mistakes but arbitrary options are passed by some users -%% as a way to identify transports. (That is, can't just do away with -%% it.) -opt(_) -> - true. +%% Anything else, which is ignored in transport config. This makes +%% options sensitive to spelling mistakes, but arbitrary options are +%% passed by some users as a way to identify transports so can't just +%% do away with it. +opt(K, _) -> + K == transport. + +%% wd/1 -wdopt({K,N}) -> +wd({K,N}) -> (K == okay orelse K == suspect) andalso is_integer(N) andalso 0 =< N; -wdopt(_) -> +wd(_) -> false. %% start_transport/2 @@ -705,16 +801,7 @@ make_config(SvcName, Opts) -> ok = encode_CER(CapOpts), - SvcOpts = make_opts((Opts -- AppOpts) -- CapOpts, - [{false, share_peers}, - {false, use_shared_peers}, - {false, monitor}, - {?NOMASK, sequence}, - {nodes, restrict_connections}, - {16#FFFFFF, incoming_maxlen}, - {true, strict_mbit}, - {true, string_decode}, - {[], spawn_opt}]), + SvcOpts = service_opts((Opts -- AppOpts) -- CapOpts), D = proplists:get_value(string_decode, SvcOpts, true), @@ -728,98 +815,22 @@ binary_caps(Caps, true) -> binary_caps(Caps, false) -> diameter_capx:binary_caps(Caps). -%% make_opts/2 - -make_opts(Opts, Defs) -> - Known = [{K, get_opt(K, Opts, D)} || {D,K} <- Defs], - Unknown = Opts -- Known, - - [] == Unknown orelse ?THROW({invalid, hd(Unknown)}), +%% service_opts/1 - [{K, opt(K,V)} || {K,V} <- Known]. - -opt(incoming_maxlen, N) - when 0 =< N, N < 1 bsl 24 -> - N; - -opt(spawn_opt, {M,F,A} = T) - when is_atom(M), is_atom(F), is_list(A) -> - T; - -opt(spawn_opt, L) - when is_list(L) -> - spawn_opts(L); - -opt(K, false = B) - when K == share_peers; - K == use_shared_peers; - K == monitor; - K == restrict_connections; - K == strict_mbit; - K == string_decode -> - B; - -opt(K, true = B) - when K == share_peers; - K == use_shared_peers; - K == strict_mbit; - K == string_decode -> - B; - -opt(restrict_connections, T) - when T == node; - T == nodes -> - T; - -opt(K, T) - when (K == share_peers - orelse K == use_shared_peers - orelse K == restrict_connections), ([] == T - orelse is_atom(hd(T))) -> - T; - -opt(monitor, P) - when is_pid(P) -> - P; - -opt(K, F) - when K == restrict_connections; - K == share_peers; - K == use_shared_peers -> - try diameter_lib:eval(F) of %% but no guarantee that it won't fail later - Nodes when is_list(Nodes) -> - F; - V -> - ?THROW({value, {K,V}}) - catch - E:R -> - ?THROW({value, {K, E, R, ?STACK}}) - end; - -opt(sequence, {_,_} = T) -> - sequence(T); - -opt(sequence = K, F) -> - try diameter_lib:eval(F) of - T -> sequence(T) - catch - E:R -> - ?THROW({value, {K, E, R, ?STACK}}) - end; - -opt(K, _) -> - ?THROW({value, K}). +service_opts(Opts) -> + Res = [setopt(service, T) || T <- Opts], + Keys = sets:to_list(sets:from_list([K || {K,_} <- Res])), %% unique + Dups = lists:foldl(fun(K,A) -> lists:keydelete(K, 1, A) end, Res, Keys), + [] == Dups orelse ?THROW({duplicate, Dups}), + Res. +%% Reject duplicates on a service, but not on a transport. There's no +%% particular reason for the inconsistency, but the historic behaviour +%% ignores all but the first of a transport_opt(), and there's no real +%% reason to change it. spawn_opts(L) -> [T || T <- L, T /= link, T /= monitor]. -sequence({H,N} = T) - when 0 =< N, N =< 32, 0 =< H, 0 == H bsr (32-N) -> - T; - -sequence(_) -> - ?THROW({value, sequence}). - make_caps(Caps, Opts) -> case diameter_capx:make_caps(Caps, Opts) of {ok, T} -> diff --git a/lib/diameter/src/base/diameter_gen.erl b/lib/diameter/src/base/diameter_gen.erl index e832832876..6add06ea38 100644 --- a/lib/diameter/src/base/diameter_gen.erl +++ b/lib/diameter/src/base/diameter_gen.erl @@ -26,6 +26,14 @@ -module(diameter_gen). +-compile({inline, [incr/8, + incr/4, + field/1, + setopts/4, + avp_arity/5, + set_failed/2, + set_strict/3]}). + -export([encode_avps/3, decode_avps/3, grouped_avp/4, @@ -37,7 +45,7 @@ -define(THROW(T), throw({?MODULE, T})). -type parent_name() :: atom(). %% parent = Message or AVP --type parent_record() :: tuple(). %% +-type parent_record() :: tuple() | avp_values() | map(). -type avp_name() :: atom(). -type avp_record() :: tuple(). -type avp_values() :: [{avp_name(), term()}]. @@ -46,17 +54,21 @@ -type grouped_avp() :: nonempty_improper_list(#diameter_avp{}, [avp()]). -type avp() :: non_grouped_avp() | grouped_avp(). +%% The arbitrary arity returned from dictionary avp_arity functions. +-define(ANY, {0, '*'}). + %% --------------------------------------------------------------------------- %% # encode_avps/3 %% --------------------------------------------------------------------------- --spec encode_avps(parent_name(), parent_record() | avp_values(), map()) +-spec encode_avps(parent_name(), parent_record(), map()) -> iolist() | no_return(). encode_avps(Name, Vals, #{module := Mod} = Opts) -> + Strict = mget(strict_arities, Opts, encode), try - encode(Name, Vals, Opts, Mod) + encode(Name, Vals, Opts, Strict, Mod) catch throw: {?MODULE, Reason} -> diameter_lib:log({encode, error}, @@ -73,128 +85,340 @@ encode_avps(Name, Vals, #{module := Mod} = Opts) -> erlang:error({encode_failure, Reason, Name, Stack}) end. -%% encode/4 +%% encode/5 -encode(Name, Vals, #{ordered_encode := false} = Opts, Mod) +encode(Name, Vals, Opts, Strict, Mod) when is_list(Vals) -> - lists:map(fun({F,V}) -> encode(Name, F, V, Opts, Mod) end, Vals); + case Opts of + #{ordered_encode := false} -> + lists:map(fun({F,V}) -> encode(Name, F, V, Opts, Strict, Mod) end, + Vals); + _ -> + Rec = Mod:'#set-'(Vals, newrec(Mod, Name)), + encode(Name, Rec, Opts, Strict, Mod) + end; -encode(Name, Vals, Opts, Mod) - when is_list(Vals) -> - encode(Name, Mod:'#set-'(Vals, newrec(Mod, Name)), Opts, Mod); +encode(Name, Map, Opts, Strict, Mod) + when is_map(Map) -> + [enc(F, A, V, Opts, Strict, Mod) || {F,A} <- Mod:avp_arity(Name), + V <- [mget(F, Map, undefined)]]; -encode(Name, Rec, Opts, Mod) -> - [encode(Name, F, V, Opts, Mod) || {F,V} <- Mod:'#get-'(Rec)]. +encode(Name, Rec, Opts, Strict, Mod) -> + [encode(Name, F, V, Opts, Strict, Mod) || {F,V} <- Mod:'#get-'(Rec)]. -%% encode/5 +%% encode/6 + +encode(_, AvpName, Values, Opts, Strict, Mod) + when Strict /= encode -> + enc(AvpName, ?ANY, Values, Opts, Strict, Mod); -encode(Name, AvpName, Values, Opts, Mod) -> - enc(Name, AvpName, Mod:avp_arity(Name, AvpName), Values, Opts, Mod). +encode(Name, AvpName, Values, Opts, Strict, Mod) -> + Arity = Mod:avp_arity(Name, AvpName), + enc(AvpName, Arity, Values, Opts, Strict, Mod). %% enc/6 -enc(_, AvpName, 1, undefined, _, _) -> +enc(AvpName, Arity, Values, Opts, Strict, Mod) + when Strict /= encode, Arity /= ?ANY -> + enc(AvpName, ?ANY, Values, Opts, Strict, Mod); + +enc(AvpName, 1, undefined, _, _, _) -> ?THROW([mandatory_avp_missing, AvpName]); -enc(Name, AvpName, 1, Value, Opts, Mod) -> - enc(Name, AvpName, [Value], Opts, Mod); +enc(AvpName, 1, Value, Opts, _, Mod) -> + H = avp_header(AvpName, Mod), + enc(AvpName, H, Value, Opts, Mod); -enc(_, _, {0,_}, [], _, _) -> +enc(_, {0,_}, [], _, _, _) -> []; -enc(_, AvpName, _, T, _, _) - when not is_list(T) -> - ?THROW([repeated_avp_as_non_list, AvpName, T]); +enc(_, _, undefined, _, _, _) -> + []; + +%% Be forgiving when a list of values is expected. If the value itself +%% is a list then the user has to wrap it to avoid each member from +%% being interpreted as an individual AVP value. +enc(AvpName, Arity, V, Opts, Strict, Mod) + when not is_list(V) -> + enc(AvpName, Arity, [V], Opts, Strict, Mod); + +enc(AvpName, {Min, Max}, Values, Opts, Strict, Mod) -> + H = avp_header(AvpName, Mod), + enc(AvpName, H, Min, 0, Max, Values, Opts, Strict, Mod). -enc(_, AvpName, {Min, _}, L, _, _) - when length(L) < Min -> - ?THROW([repeated_avp_insufficient_arity, AvpName, Min, L]); +%% enc/9 -enc(_, AvpName, {_, Max}, L, _, _) - when Max < length(L) -> - ?THROW([repeated_avp_excessive_arity, AvpName, Max, L]); +enc(AvpName, H, Min, N, Max, Vs, Opts, Strict, Mod) + when Strict /= encode; + Max == '*', Min =< N -> + [enc(AvpName, H, V, Opts, Mod) || V <- Vs]; + +enc(AvpName, _, Min, N, _, [], _, _, _) + when N < Min -> + ?THROW([repeated_avp_insufficient_arity, AvpName, Min, N]); + +enc(_, _, _, _, _, [], _, _, _) -> + []; -enc(Name, AvpName, _, Values, Opts, Mod) -> - enc(Name, AvpName, Values, Opts, Mod). +enc(AvpName, _, _, N, Max, _, _, _, _) + when Max =< N -> + ?THROW([repeated_avp_excessive_arity, AvpName, Max]); + +enc(AvpName, H, Min, N, Max, [V|Vs], Opts, Strict, Mod) -> + [enc(AvpName, H, V, Opts, Mod) + | enc(AvpName, H, Min, N+1, Max, Vs, Opts, Strict, Mod)]. + +%% avp_header/2 + +avp_header('AVP', _) -> + false; + +avp_header(AvpName, Mod) -> + {_,_,_} = Mod:avp_header(AvpName). %% enc/5 -enc(Name, 'AVP', Values, Opts, Mod) -> - [enc_AVP(Name, A, Opts, Mod) || A <- Values]; +enc('AVP', false, Value, Opts, Mod) -> + enc_AVP(Value, Opts, Mod); -enc(_, AvpName, Values, Opts, Mod) -> - enc(AvpName, Values, Opts, Mod). +enc(AvpName, Hdr, Value, Opts, Mod) -> + enc1(AvpName, Hdr, Value, Opts, Mod). -%% enc/4 +%% enc1/5 -enc(AvpName, Values, Opts, Mod) -> - H = Mod:avp_header(AvpName), - [diameter_codec:pack_data(H, Mod:avp(encode, V, AvpName, Opts)) - || V <- Values]. +enc1(AvpName, {_,_,_} = Hdr, Value, Opts, Mod) -> + diameter_codec:pack_data(Hdr, Mod:avp(encode, Value, AvpName, Opts)). -%% enc_AVP/4 +%% enc1/6 + +enc1(AvpName, {_,_,_} = Hdr, Value, Opts, Mod, Dict) -> + diameter_codec:pack_data(Hdr, avp(encode, Value, AvpName, Opts, Mod, Dict)). + +%% enc_AVP/3 %% No value: assume AVP data is already encoded. The normal case will %% be when this is passed back from #diameter_packet.errors as a %% consequence of a failed decode. Any AVP can be encoded this way %% however, which side-steps any arity checks for known AVP's and %% could potentially encode something unfortunate. -enc_AVP(_, #diameter_avp{value = undefined} = A, Opts, _) -> +enc_AVP(#diameter_avp{value = undefined} = A, Opts, _) -> diameter_codec:pack_avp(A, Opts); -%% Missing name for value encode. -enc_AVP(_, #diameter_avp{name = N, value = V}, _, _) - when N == undefined; - N == 'AVP' -> - ?THROW([value_with_nameless_avp, N, V]); +%% Encode a name/value pair using an alternate dictionary if need be ... +enc_AVP(#diameter_avp{name = AvpName, value = Value}, Opts, Mod) -> + enc_AVP(AvpName, Value, Opts, Mod); +enc_AVP({AvpName, Value}, Opts, Mod) -> + enc_AVP(AvpName, Value, Opts, Mod); + +%% ... or with a specified dictionary. +enc_AVP({Dict, AvpName, Value}, Opts, Mod) -> + enc1(AvpName, Dict:avp_header(AvpName), Value, Opts, Mod, Dict). -%% Or not. Ensure that 'AVP' is the appropriate field. Note that if we -%% don't know this AVP at all then the encode will fail. -enc_AVP(Name, #diameter_avp{name = AvpName, value = Data}, Opts, Mod) -> - 0 == Mod:avp_arity(Name, AvpName) - orelse ?THROW([known_avp_as_AVP, Name, AvpName, Data]), - enc(AvpName, [Data], Opts, Mod); +%% Don't guard against anything being sent as a generic 'AVP', which +%% allows arity restrictions to be abused. -%% The backdoor ... -enc_AVP(_, {AvpName, Value}, Opts, Mod) -> - enc(AvpName, [Value], Opts, Mod); +%% enc_AVP/4 -%% ... and the side door. -enc_AVP(_Name, {_Dict, _AvpName, _Data} = T, Opts, _) -> - diameter_codec:pack_avp(#diameter_avp{data = T}, Opts). +enc_AVP(AvpName, Value, Opts, Mod) -> + try Mod:avp_header(AvpName) of + H -> + enc1(AvpName, H, Value, Opts, Mod) + catch + error: _ -> + Dicts = mget(avp_dictionaries, Opts, []), + enc_AVP(Dicts, AvpName, Value, Opts, Mod) + end. + +%% enc_AVP/5 + +enc_AVP([Dict | Rest], AvpName, Value, Opts, Mod) -> + try Dict:avp_header(AvpName) of + H -> + enc1(AvpName, H, Value, Opts, Mod, Dict) + catch + error: _ -> + enc_AVP(Rest, AvpName, Value, Opts, Mod) + end; + +enc_AVP([], AvpName, _, _, _) -> + ?THROW([no_dictionary, AvpName]). %% --------------------------------------------------------------------------- %% # decode_avps/3 %% --------------------------------------------------------------------------- --spec decode_avps(parent_name(), [#diameter_avp{}], map()) - -> {parent_record(), [avp()], Failed} +-spec decode_avps(parent_name(), binary(), map()) + -> {parent_record() | parent_name(), [avp()], Failed} when Failed :: [{5000..5999, #diameter_avp{}}]. -decode_avps(Name, Recs, #{module := Mod} = Opts) -> - {Avps, {Rec, Failed}} - = mapfoldl(fun(T,A) -> decode(Name, Opts, Mod, T, A) end, - {newrec(Mod, Name), []}, - Recs), - {Rec, Avps, Failed ++ missing(Rec, Name, Failed, Opts, Mod)}. -%% Append 5005 errors so that errors are reported in the order +decode_avps(Name, Bin, #{module := Mod, decode_format := Fmt} = Opts) -> + Strict = mget(strict_arities, Opts, decode), + [AM, Avps, Failed | Rec] + = decode(Bin, Name, Mod, Fmt, Strict, Opts, 0, #{}), + %% AM counts the number of top-level AVPs, which missing/5 then + %% uses when appending 5005 errors. + {reformat(Name, Rec, Strict, Mod, Fmt), + Avps, + Failed ++ missing(Name, Strict, Mod, Opts, AM)}. + +%% Append arity errors so that errors are reported in the order %% encountered. Failed-AVP should typically contain the first -%% encountered error accordg to the RFC. +%% error encountered. + +%% decode/8 + +decode(<<Code:32, V:1, M:1, P:1, _:5, Len:24, I:V/unit:32, Rest/binary>>, + Name, + Mod, + Fmt, + Strict, + Opts, + Idx, + AM) -> + decode(Rest, + Code, + if 1 == V -> I; true -> undefined end, + Len - 8 - 4*V, %% possibly negative, causing case match to fail + (4 - (Len rem 4)) rem 4, + 1 == M, + 1 == P, + Name, + Mod, + Fmt, + Strict, + Opts, + Idx, + AM); + +decode(<<>>, Name, Mod, Fmt, Strict, _, _, AM) -> + [AM, [], [] | newrec(Fmt, Mod, Name, Strict)]; + +decode(Bin, Name, Mod, Fmt, Strict, _, Idx, AM) -> + Avp = #diameter_avp{data = Bin, index = Idx}, + [AM, [Avp], [{5014, Avp}] | newrec(Fmt, Mod, Name, Strict)]. + +%% decode/14 + +decode(Bin, Code, Vid, DataLen, Pad, M, P, Name, Mod, Fmt, Strict, Opts0, + Idx, AM0) -> + case Bin of + <<Data:DataLen/binary, _:Pad/binary, T/binary>> -> + {NameT, Field, Arity, {I, AM}} + = incr(Name, Code, Vid, M, Mod, Strict, Opts0, AM0), + + Opts = setopts(NameT, Name, M, Opts0), + %% Not AvpName or else a failed Failed-AVP + %% decode is packed into 'AVP'. + + Avp = #diameter_avp{code = Code, + vendor_id = Vid, + is_mandatory = M, + need_encryption = P, + data = Data, + name = name(NameT), + type = type(NameT), + index = Idx}, + + Dec = dec(Data, Name, NameT, Mod, Fmt, Opts, Avp), + Acc = decode(T, Name, Mod, Fmt, Strict, Opts, Idx+1, AM),%% recurse + acc(Acc, Dec, I, Field, Arity, Strict, Mod, Opts); + _ -> + {NameT, _Field, _Arity, {_, AM}} + = incr(Name, Code, Vid, M, Mod, Strict, Opts0, AM0), + + Avp = #diameter_avp{code = Code, + vendor_id = Vid, + is_mandatory = M, + need_encryption = P, + data = Bin, + name = name(NameT), + type = type(NameT), + index = Idx}, + + [AM, [Avp], [{5014, Avp}] | newrec(Fmt, Mod, Name, Strict)] + end. + +%% incr/8 + +incr(Name, Code, Vid, M, Mod, Strict, Opts, AM0) -> + NameT = Mod:avp_name(Code, Vid), %% {AvpName, Type} | 'AVP' + Field = field(NameT), %% AvpName | 'AVP' + Arity = avp_arity(Name, Field, Mod, Opts, M), + if 0 == Arity, 'AVP' /= Field -> + A = pack_arity(Name, Field, Opts, Mod, M), + {NameT, 'AVP', A, incr('AVP', A, Strict, AM0)}; + true -> + {NameT, Field, Arity, incr(Field, Arity, Strict, AM0)} + end. -%% mapfoldl/3 +%% Data is a truncated header if command_code = undefined, otherwise +%% payload bytes. The former is padded to the length of a header if +%% the AVP reaches an outgoing encode. %% -%% Like lists:mapfoldl/3, but don't reverse the list. +%% RFC 6733 says that an AVP returned with 5014 can contain a minimal +%% payload for the AVP's type, but don't always know the type. -mapfoldl(F, Acc, List) -> - mapfoldl(F, Acc, List, []). +setopts('AVP', _, _, Opts) -> + Opts; -mapfoldl(F, Acc0, [T|Rest], List) -> - {B, Acc} = F(T, Acc0), - mapfoldl(F, Acc, Rest, [B|List]); -mapfoldl(_, Acc, [], List) -> - {List, Acc}. +setopts({_, Type}, Name, M, Opts) -> + set_failed(Name, set_strict(Type, M, Opts)). -%% 3588: +%% incr/4 + +incr(_, A, SA, AM) + when A == ?ANY; + A == 0; + SA /= decode -> + {undefined, AM}; + +incr(AvpName, _, _, AM) -> + case AM of + #{AvpName := N} -> + {N, AM#{AvpName => N+1}}; + _ -> + {0, AM#{AvpName => 1}} + end. + +%% mget/3 +%% +%% Measurably faster than maps:get/3. + +mget(Key, Map, Def) -> + case Map of + #{Key := V} -> + V; + _ -> + Def + end. + +%% name/1 + +name({Name, _}) -> + Name; +name(_) -> + undefined. + +%% type/1 + +type({_, Type}) -> + Type; +type(_) -> + undefined. + +%% missing/5 + +missing(Name, decode, Mod, Opts, AM) -> + [{5005, empty_avp(N, Opts, Mod)} || {N,A} <- Mod:avp_arity(Name), + N /= 'AVP', + Mn <- [min_arity(A)], + 0 < Mn, + mget(N, AM, 0) < Mn]; + +missing(_, _, _, _, _) -> + []. + +%% 3588/6733: %% %% DIAMETER_MISSING_AVP 5005 %% The request did not contain an AVP that is required by the Command @@ -204,57 +428,20 @@ mapfoldl(_, Acc, [], List) -> %% Vendor-Id if applicable. The value field of the missing AVP %% should be of correct minimum length and contain zeros. -missing(Rec, Name, Failed, Opts, Mod) -> - Avps = lists:foldl(fun({_, #diameter_avp{code = C, vendor_id = V}}, A) -> - maps:put({C,V}, true, A) - end, - maps:new(), - Failed), - missing(Mod:avp_arity(Name), tl(tuple_to_list(Rec)), Avps, Opts, Mod, []). - -missing([{Name, Arity} | As], [Value | Vs], Avps, Opts, Mod, Acc) -> - missing(As, - Vs, - Avps, - Opts, - Mod, - case - [H || missing_arity(Arity, Value), - {C,_,V} = H <- [Mod:avp_header(Name)], - not maps:is_key({C,V}, Avps)] - of - [H] -> - [{5005, empty_avp(Name, H, Opts, Mod)} | Acc]; - [] -> - Acc - end); - -missing([], [], _, _, _, Acc) -> - Acc. - -%% Maximum arities have already been checked in building the record. - -missing_arity(1, V) -> - V == undefined; -missing_arity({0, _}, _) -> - false; -missing_arity({1, _}, L) -> - [] == L; -missing_arity({Min, _}, L) -> - not has_prefix(Min, L). - -%% Compare a non-negative integer and the length of a list without -%% computing the length. -has_prefix(0, _) -> - true; -has_prefix(_, []) -> - false; -has_prefix(N, [_|L]) -> - has_prefix(N-1, L). +%% min_arity/1 -%% empty_avp/4 +min_arity(1) -> + 1; +min_arity({Mn,_}) -> + Mn. -empty_avp(Name, {Code, Flags, VId}, Opts, Mod) -> +%% empty_avp/3 + +empty_avp('AVP', _, _) -> + #diameter_avp{data = <<0:64>>}; + +empty_avp(Name, Opts, Mod) -> + {Code, Flags, VId} = Mod:avp_header(Name), {Name, Type} = Mod:avp_name(Code, VId), #diameter_avp{name = Name, code = Code, @@ -273,21 +460,22 @@ empty_avp(Name, {Code, Flags, VId}, Opts, Mod) -> %% specific errors that can be described by this AVP are described in %% the following section. -%% decode/5 +%% field/1 -decode(Name, - Opts, - Mod, - #diameter_avp{code = Code, vendor_id = Vid} - = Avp, - Acc) -> - decode(Name, Opts, Mod, Mod:avp_name(Code, Vid), Avp, Acc). +field({AvpName, _}) -> + AvpName; +field(_) -> + 'AVP'. + +%% dec/7 + +%% AVP not in dictionary: try an alternate. -%% decode/6 +dec(_, _, 'AVP', _Mod, none, _, Avp) -> %% none decode is no-op + Avp; -%% AVP not in dictionary. -decode(Name, Opts, Mod, 'AVP', Avp, Acc) -> - decode_AVP(Name, Avp, Opts, Mod, Acc); +dec(Data, Name, 'AVP', Mod, Fmt, Opts, Avp) -> + dec_AVP(dicts(Mod, Opts), Data, Name, Mod, Fmt, Opts, Avp); %% 6733, 4.4: %% @@ -336,130 +524,130 @@ decode(Name, Opts, Mod, 'AVP', Avp, Acc) -> %% defined the RFC's "unrecognized", which is slightly stronger than %% "not defined".) -decode(Name, Opts0, Mod, {AvpName, Type}, Avp, Acc) -> - #diameter_avp{data = Data, is_mandatory = M} - = Avp, +dec(Data, Name, {AvpName, Type}, Mod, Fmt, Opts, Avp) -> + #{app_dictionary := AppMod, failed_avp := Failed} + = Opts, - %% Whether or not to ignore an M-bit on an encapsulated AVP, or on - %% all AVPs with the service_opt() strict_mbit. - Opts1 = set_strict(Type, M, Opts0), + %% Reset the dictionary for best-effort decode of Failed-AVP. + Dict = if Failed -> AppMod; + true -> Mod + end, - %% Whether or not we're decoding within Failed-AVP and should - %% ignore decode errors. - #{dictionary := AppMod, failed_avp := Failed} - = Opts - = set_failed(Name, Opts1), %% Not AvpName or else a failed Failed-AVP - %% decode is packed into 'AVP'. + dec(Data, Name, AvpName, Type, Mod, Dict, Fmt, Failed, Opts, Avp). - %% Reset the dictionary for best-effort decode of Failed-AVP. - DecMod = if Failed -> - AppMod; - true -> - Mod - end, - - %% On decode, a Grouped AVP is represented as a #diameter_avp{} - %% list with AVP as head and component AVPs as tail. On encode, - %% data can be a list of component AVPs. - - try avp_decode(Data, AvpName, Opts, DecMod, Mod) of - {Rec, As} when Type == 'Grouped' -> - A = Avp#diameter_avp{name = AvpName, - value = Rec, - type = Type}, - {[A|As], pack_avp(Name, A, Opts, Mod, Acc)}; +%% dicts/2 - V when Type /= 'Grouped' -> - A = Avp#diameter_avp{name = AvpName, - value = V, - type = Type}, - {A, pack_avp(Name, A, Opts, Mod, Acc)} - catch - throw: {?MODULE, {grouped, Error, ComponentAvps}} -> - decode_error(Name, - Error, - ComponentAvps, - Opts, - Mod, - Avp#diameter_avp{name = AvpName, - data = trim(Avp#diameter_avp.data), - type = Type}, - Acc); +dicts(Mod, #{app_dictionary := Mod, avp_dictionaries := Dicts}) -> + Dicts; + +dicts(_, #{app_dictionary := Dict, avp_dictionaries := Dicts}) -> + [Dict | Dicts]; +dicts(Mod, #{app_dictionary := Mod}) -> + []; + +dicts(_, #{app_dictionary := Dict}) -> + [Dict]. + +%% dec/10 + +dec(Data, Name, AvpName, Type, Mod, Dict, Fmt, Failed, Opts, Avp) -> + try avp(decode, Data, AvpName, Opts, Mod, Dict) of + V -> + set(Type, Fmt, Avp, V) + catch + throw: {?MODULE, T} -> + decode_error(Failed, Fmt, T, Avp); error: Reason -> - decode_error(Name, - Reason, - Opts, - Mod, - Avp#diameter_avp{name = AvpName, - data = trim(Avp#diameter_avp.data), - type = Type}, - Acc) + decode_error(Failed, Reason, Name, Mod, Opts, Avp) end. -%% avp_decode/5 +%% dec_AVP/7 -avp_decode(Data, AvpName, Opts, Mod, Mod) -> - Mod:avp(decode, Data, AvpName, Opts); +dec_AVP([], _, _, _, _, _, Avp) -> + Avp; -avp_decode(Data, AvpName, Opts, Mod, _) -> - Mod:avp(decode, Data, AvpName, Opts, Mod). +dec_AVP(Dicts, Data, Name, Mod, Fmt, Opts, #diameter_avp{code = Code, + vendor_id = Vid} + = Avp) -> + dec_AVP(Dicts, Data, Name, Mod, Fmt, Opts, Code, Vid, Avp). -%% trim/1 +%% dec_AVP/9 %% -%% Remove any extra bit that was added in diameter_codec to induce a -%% 5014 error. +%% Try to decode an AVP in the first alternate dictionary that defines +%% it. + +dec_AVP([Dict | Rest], Data, Name, Mod, Fmt, Opts, Code, Vid, Avp) -> + case Dict:avp_name(Code, Vid) of + {AvpName, Type} -> + A = Avp#diameter_avp{name = AvpName, + type = Type}, + #{failed_avp := Failed} = Opts, + dec(Data, Name, AvpName, Type, Mod, Dict, Fmt, Failed, Opts, A); + _ -> + dec_AVP(Rest, Data, Name, Mod, Fmt, Opts, Code, Vid, Avp) + end; -trim(#diameter_avp{data = Data} = Avp) -> - Avp#diameter_avp{data = trim(Data)}; +dec_AVP([], _, _, _, _, _, _, _, Avp) -> + Avp. -trim({5014, Bin}) -> - Bin; +%% set/4 +%% +%% A Grouped AVP is represented as a #diameter_avp{} list with AVP +%% as head and component AVPs as tail. -trim(Avps) - when is_list(Avps) -> - lists:map(fun trim/1, Avps); +set('Grouped', none, Avp, V) -> + {_Rec, As} = V, + [Avp | As]; -trim(Avp) -> - Avp. +set('Grouped', _, Avp, V) -> + {Rec, As} = V, + [Avp#diameter_avp{value = Rec} | As]; -%% decode_error/7 +set(_, _, Avp, V) -> + Avp#diameter_avp{value = V}. -decode_error(Name, [_ | Rec], _, #{failed_avp := true} = Opts, Mod, Avp, Acc) -> - decode_AVP(Name, Avp#diameter_avp{value = Rec}, Opts, Mod, Acc); +%% decode_error/4 +%% +%% Error when decoding a grouped AVP. -decode_error(Name, _, _, #{failed_avp := true} = Opts, Mod, Avp, Acc) -> - decode_AVP(Name, Avp, Opts, Mod, Acc); +decode_error(true, none, _, Avp) -> + Avp; -decode_error(_, [Error | _], ComponentAvps, _, _, Avp, Acc) -> - decode_error(Error, Avp, Acc, ComponentAvps); +decode_error(true, _, {Rec, _, _}, Avp) -> + Avp#diameter_avp{value = Rec}; -decode_error(_, Error, ComponentAvps, _, _, Avp, Acc) -> - decode_error(Error, Avp, Acc, ComponentAvps). +decode_error(false, _, {_, ComponentAvps, [{RC,A} | _]}, Avp) -> + {RC, [Avp | ComponentAvps], Avp#diameter_avp{data = [A]}}. -%% decode_error/5 +%% decode_error/6 +%% +%% Error when decoding a non-grouped AVP. -decode_error(Name, _Reason, #{failed_avp := true} = Opts, Mod, Avp, Acc) -> - decode_AVP(Name, Avp, Opts, Mod, Acc); +decode_error(true, _, _, _, _, Avp) -> + Avp; -decode_error(Name, Reason, Opts, Mod, Avp, {Rec, Failed}) -> +decode_error(false, Reason, Name, Mod, Opts, Avp) -> Stack = diameter_lib:get_stacktrace(), diameter_lib:log(decode_error, ?MODULE, ?LINE, {Reason, Name, Avp#diameter_avp.name, Mod, Stack}), - {Avp, {Rec, [rc(Reason, Avp, Opts, Mod) | Failed]}}. + rc(Reason, Avp, Opts, Mod). -%% decode_error/4 +%% avp/6 -decode_error({RC, ErrorData}, Avp, {Rec, Failed}, ComponentAvps) -> - E = Avp#diameter_avp{data = [ErrorData]}, - {[Avp | trim(ComponentAvps)], {Rec, [{RC, E} | Failed]}}. +avp(T, Data, AvpName, Opts, Mod, Mod) -> + Mod:avp(T, Data, AvpName, Opts); -%% set_strict/3 +avp(T, Data, AvpName, Opts, _, Mod) -> + Mod:avp(T, Data, AvpName, Opts#{module := Mod}). +%% set_strict/3 +%% %% Set false as soon as we see a Grouped AVP that doesn't set the %% M-bit, to ignore the M-bit on an encapsulated AVP. + set_strict('Grouped', false = M, #{strict_mbit := true} = Opts) -> Opts#{strict_mbit := M}; set_strict(_, _, Opts) -> @@ -476,185 +664,207 @@ set_failed('Failed-AVP', #{failed_avp := false} = Opts) -> set_failed(_, Opts) -> Opts. -%% decode_AVP/5 -%% -%% Don't know this AVP: see if it can be packed in an 'AVP' field -%% undecoded. Note that the type field is 'undefined' in this case. - -decode_AVP(Name, Avp, Opts, Mod, Acc) -> - {trim(Avp), pack_AVP(Name, Avp, Opts, Mod, Acc)}. - -%% rc/2 - -%% diameter_types will raise an error of this form to communicate -%% DIAMETER_INVALID_AVP_LENGTH (5014). A module specified to a -%% @custom_types tag in a dictionary file can also raise an error of -%% this form. -rc({'DIAMETER', 5014 = RC, _}, #diameter_avp{name = AvpName} = Avp, Opts, Mod) -> - {RC, Avp#diameter_avp{data = Mod:empty_value(AvpName, Opts)}}; +%% acc/8 + +acc([AM | Acc], As, I, Field, Arity, Strict, Mod, Opts) -> + [AM | acc1(Acc, As, I, Field, Arity, Strict, Mod, Opts)]. + +%% acc1/8 + +%% Faulty AVP, not grouped. +acc1(Acc, {_RC, Avp} = E, _, _, _, _, _, _) -> + [Avps, Failed | Rec] = Acc, + [[Avp | Avps], [E | Failed] | Rec]; + +%% Faulty component in grouped AVP. +acc1(Acc, {RC, As, Avp}, _, _, _, _, _, _) -> + [Avps, Failed | Rec] = Acc, + [[As | Avps], [{RC, Avp} | Failed] | Rec]; + +%% Grouped AVP ... +acc1([Avps | Acc], [Avp|_] = As, I, Field, Arity, Strict, Mod, Opts) -> + [[As|Avps] | acc2(Acc, Avp, I, Field, Arity, Strict, Mod, Opts)]; + +%% ... or not. +acc1([Avps | Acc], Avp, I, Field, Arity, Strict, Mod, Opts) -> + [[Avp|Avps] | acc2(Acc, Avp, I, Field, Arity, Strict, Mod, Opts)]. + +%% The component list of a Grouped AVP is discarded when packing into +%% the record (or equivalent): the values in an 'AVP' field are +%% diameter_avp records, not a list of records in the Grouped case, +%% and the decode into the value field is best-effort. The reason is +%% history more than logic: it would probably have made more sense to +%% retain the same structure as in diameter_packet.avps, but an 'AVP' +%% list has always been flat. + +%% acc2/8 + +%% No errors, but nowhere to pack. +acc2(Acc, Avp, _, 'AVP', 0, _, _, _) -> + [Failed | Rec] = Acc, + [[{rc(Avp), Avp} | Failed] | Rec]; + +%% Relaxed arities. +acc2(Acc, Avp, _, Field, Arity, Strict, Mod, _) + when Strict /= decode -> + pack(Arity, Field, Avp, Mod, Acc); + +%% No maximum arity. +acc2(Acc, Avp, _, Field, {_,'*'} = Arity, _, Mod, _) -> + pack(Arity, Field, Avp, Mod, Acc); + +%% Or check. +acc2(Acc, Avp, I, Field, Arity, _, Mod, _) -> + Mx = max_arity(Arity), + if Mx =< I -> + [Failed | Rec] = Acc, + [[{5009, Avp} | Failed] | Rec]; + true -> + pack(Arity, Field, Avp, Mod, Acc) + end. -%% 3588: +%% 3588/6733: %% -%% DIAMETER_INVALID_AVP_VALUE 5004 -%% The request contained an AVP with an invalid value in its data -%% portion. A Diameter message indicating this error MUST include -%% the offending AVPs within a Failed-AVP AVP. -rc(_, Avp, _, _) -> - {5004, Avp}. - -%% pack_avp/5 - -pack_avp(Name, #diameter_avp{name = AvpName} = Avp, Opts, Mod, Acc) -> - pack_avp(Name, Mod:avp_arity(Name, AvpName), Avp, Opts, Mod, Acc). +%% DIAMETER_AVP_OCCURS_TOO_MANY_TIMES 5009 +%% A message was received that included an AVP that appeared more +%% often than permitted in the message definition. The Failed-AVP +%% AVP MUST be included and contain a copy of the first instance of +%% the offending AVP that exceeded the maximum number of occurrences -%% pack_avp/6 +%% max_arity/1 -pack_avp(Name, 0, Avp, Opts, Mod, Acc) -> - pack_AVP(Name, Avp, Opts, Mod, Acc); +max_arity(1) -> + 1; +max_arity({_,Mx}) -> + Mx. -pack_avp(_, Arity, #diameter_avp{name = AvpName} = Avp, _Opts, Mod, Acc) -> - pack(Arity, AvpName, Avp, Mod, Acc). +%% rc/1 -%% pack_AVP/5 +rc(#diameter_avp{is_mandatory = M}) -> + if M -> 5001; true -> 5008 end. -%% Length failure was induced because of a header/payload length -%% mismatch. The AVP Length is reset to match the received data if -%% this AVP is encoded in an answer message, since the length is -%% computed. +%% 3588: %% -%% Data is a truncated header if command_code = undefined, otherwise -%% payload bytes. The former is padded to the length of a header if -%% the AVP reaches an outgoing encode in diameter_codec. +%% DIAMETER_AVP_UNSUPPORTED 5001 +%% The peer received a message that contained an AVP that is not +%% recognized or supported and was marked with the Mandatory bit. A +%% Diameter message with this error MUST contain one or more Failed- +%% AVP AVP containing the AVPs that caused the failure. %% -%% RFC 6733 says that an AVP returned with 5014 can contain a minimal -%% payload for the AVP's type, but in this case we don't know the -%% type. - -pack_AVP(_, #diameter_avp{data = {5014 = RC, Data}} = Avp, _, _, Acc) -> - {Rec, Failed} = Acc, - {Rec, [{RC, Avp#diameter_avp{data = Data}} | Failed]}; - -pack_AVP(Name, Avp, Opts, Mod, Acc) -> - pack_arity(Name, pack_arity(Name, Opts, Mod, Avp), Avp, Mod, Acc). +%% DIAMETER_AVP_NOT_ALLOWED 5008 +%% A message was received with an AVP that MUST NOT be present. The +%% Failed-AVP AVP MUST be included and contain a copy of the +%% offending AVP. %% pack_arity/5 -pack_arity(_, 0, #diameter_avp{is_mandatory = M} = Avp, _, Acc) -> - {Rec, Failed} = Acc, - {Rec, [{if M -> 5001; true -> 5008 end, Avp} | Failed]}; - -pack_arity(_, Arity, Avp, Mod, Acc) -> - pack(Arity, 'AVP', Avp, Mod, Acc). - %% Give Failed-AVP special treatment since (1) it'll contain any %% unrecognized mandatory AVP's and (2) the RFC 3588 grammar failed to %% allow for Failed-AVP in an answer-message. -pack_arity(Name, - #{strict_mbit := Strict, - failed_avp := Failed}, - Mod, - #diameter_avp{is_mandatory = M, - name = AvpName}) -> +pack_arity(Name, AvpName, _, Mod, M) + when Name == 'Failed-AVP'; + Name == 'answer-message', AvpName == 'Failed-AVP'; + not M -> + Mod:avp_arity(Name, 'AVP'); +%% Not testing just Name /= 'Failed-AVP' means we're changing the +%% packing of AVPs nested within Failed-AVP, but the point of +%% ignoring errors within Failed-AVP is to decode as much as +%% possible, and failing because a mandatory AVP couldn't be +%% packed into a dedicated field defeats that point. - %% Not testing just Name /= 'Failed-AVP' means we're changing the - %% packing of AVPs nested within Failed-AVP, but the point of - %% ignoring errors within Failed-AVP is to decode as much as - %% possible, and failing because a mandatory AVP couldn't be - %% packed into a dedicated field defeats that point. +pack_arity(Name, _, #{strict_mbit := Strict, failed_avp := Failed}, Mod, _) + when not Strict; + Failed -> + Mod:avp_arity(Name, 'AVP'); - if Failed == true; - Name == 'Failed-AVP'; - Name == 'answer-message', AvpName == 'Failed-AVP'; - not M; - not Strict -> - Mod:avp_arity(Name, 'AVP'); - true -> - 0 - end. +pack_arity(_, _, _, _, _) -> + 0. + +%% avp_arity/5 + +avp_arity(Name, 'AVP' = AvpName, Mod, Opts, M) -> + pack_arity(Name, AvpName, Opts, Mod, M); + +avp_arity(Name, AvpName, Mod, _, _) -> + Mod:avp_arity(Name, AvpName). + +%% rc/4 + +%% Length error communicated from diameter_types or a +%% @custom_types/@codecs module. +rc({'DIAMETER', 5014 = RC, _}, #diameter_avp{name = AvpName} = A, Opts, Mod) -> + {RC, A#diameter_avp{data = Mod:empty_value(AvpName, Opts)}}; %% 3588: %% -%% DIAMETER_AVP_UNSUPPORTED 5001 -%% The peer received a message that contained an AVP that is not -%% recognized or supported and was marked with the Mandatory bit. A -%% Diameter message with this error MUST contain one or more Failed- -%% AVP AVP containing the AVPs that caused the failure. -%% -%% DIAMETER_AVP_NOT_ALLOWED 5008 -%% A message was received with an AVP that MUST NOT be present. The -%% Failed-AVP AVP MUST be included and contain a copy of the -%% offending AVP. +%% DIAMETER_INVALID_AVP_VALUE 5004 +%% The request contained an AVP with an invalid value in its data +%% portion. A Diameter message indicating this error MUST include +%% the offending AVPs within a Failed-AVP AVP. +rc(_, Avp, _, _) -> + {5004, Avp}. %% pack/5 -pack(Arity, FieldName, Avp, Mod, {Rec, _} = Acc) -> - pack(Mod:'#get-'(FieldName, Rec), Arity, FieldName, Avp, Mod, Acc). +pack(Arity, F, Avp, Mod, [Failed | Rec]) -> + [Failed | set(Arity, F, value(F, Avp), Mod, Rec)]. -%% pack/6 +%% set/5 -pack(undefined, 1, 'AVP' = F, Avp, Mod, {Rec, Failed}) -> %% unlikely - {Mod:'#set-'({F, Avp}, Rec), Failed}; +set(_, _, _, _, None) + when is_atom(None) -> + None; -pack(undefined, 1, F, #diameter_avp{value = V}, Mod, {Rec, Failed}) -> - {Mod:'#set-'({F, V}, Rec), Failed}; +set(1, F, Value, _, Map) + when is_map(Map) -> + Map#{F => Value}; -%% 3588: -%% -%% DIAMETER_AVP_OCCURS_TOO_MANY_TIMES 5009 -%% A message was received that included an AVP that appeared more -%% often than permitted in the message definition. The Failed-AVP -%% AVP MUST be included and contain a copy of the first instance of -%% the offending AVP that exceeded the maximum number of occurrences -%% +set(_, F, V, _, Map) + when is_map(Map) -> + maps:update_with(F, fun(Vs) -> [V|Vs] end, [V], Map); -pack(_, 1, _, Avp, _, {Rec, Failed}) -> - {Rec, [{5009, Avp} | Failed]}; - -pack(L, {_, Max}, F, Avp, Mod, {Rec, Failed}) -> - case '*' /= Max andalso has_prefix(Max+1, L) of - true -> - {Rec, [{5009, Avp} | Failed]}; - false when F == 'AVP' -> - {Mod:'#set-'({F, [Avp | L]}, Rec), Failed}; - false -> - {Mod:'#set-'({F, [Avp#diameter_avp.value | L]}, Rec), Failed} - end. +set(1, F, Value, Mod, Rec) -> + Mod:'#set-'({F, Value}, Rec); + +set(_, F, V, Mod, Rec) -> + Vs = Mod:'#get-'(F, Rec), + Mod:'#set-'({F, [V|Vs]}, Rec). + +%% value/2 + +value('AVP', Avp) -> + Avp; + +value(_, #diameter_avp{value = V}) -> + V. %% --------------------------------------------------------------------------- %% # grouped_avp/3 %% --------------------------------------------------------------------------- --spec grouped_avp(decode, avp_name(), binary() | {5014, binary()}, term()) +%% Note that Grouped is the only AVP type that doesn't just return a +%% decoded value, also returning the list of component diameter_avp +%% records. + +-spec grouped_avp(decode, avp_name(), binary(), term()) -> {avp_record(), [avp()]}; (encode, avp_name(), avp_record() | avp_values(), term()) -> iolist() | no_return(). -%% Length error induced by diameter_codec:collect_avps/1: the AVP -%% length in the header was too short (insufficient for the extracted -%% header) or too long (past the end of the message). An empty payload -%% is sufficient according to the RFC text for 5014. -grouped_avp(decode, _Name, {5014 = RC, _Bin}, _) -> - ?THROW({grouped, {RC, []}, []}); - -grouped_avp(decode, Name, Data, Opts) -> - grouped_decode(Name, diameter_codec:collect_avps(Data), Opts); +%% An error in decoding a component AVP throws the first faulty +%% component, which a catch wraps in the Grouped AVP in question. A +%% partially decoded record is only used when ignoring errors in +%% Failed-AVP. +grouped_avp(decode, Name, Bin, Opts) -> + {Rec, Avps, Es} = T = decode_avps(Name, Bin, Opts), + [] == Es orelse ?THROW(T), + {Rec, Avps}; grouped_avp(encode, Name, Data, Opts) -> encode_avps(Name, Data, Opts). -%% grouped_decode/2 -%% -%% Note that Grouped is the only AVP type that doesn't just return a -%% decoded value, also returning the list of component diameter_avp -%% records. - -%% Length error in trailing component AVP. -grouped_decode(_Name, {Error, Acc}, _) -> - {5014, Avp} = Error, - ?THROW({grouped, Error, [Avp | Acc]}); - %% 7.5. Failed-AVP AVP %% In the case where the offending AVP is embedded within a Grouped AVP, @@ -665,15 +875,6 @@ grouped_decode(_Name, {Error, Acc}, _) -> %% to the single offending AVP. This enables the recipient to detect %% the location of the offending AVP when embedded in a group. -%% An error in decoding a component AVP throws the first faulty -%% component, which the catch in d/3 wraps in the Grouped AVP in -%% question. A partially decoded record is only used when ignoring -%% errors in Failed-AVP. -grouped_decode(Name, ComponentAvps, Opts) -> - {Rec, Avps, Es} = decode_avps(Name, ComponentAvps, Opts), - [] == Es orelse ?THROW({grouped, [{_,_} = hd(Es) | Rec], Avps}), - {Rec, Avps}. - %% --------------------------------------------------------------------------- %% # empty_group/2 %% --------------------------------------------------------------------------- @@ -705,5 +906,45 @@ empty(Name, #{module := Mod} = Opts) -> %% ------------------------------------------------------------------------------ +%% newrec/4 + +newrec(none, _, Name, _) -> + Name; + +newrec(record, Mod, Name, T) + when T /= decode -> + RecName = Mod:name2rec(Name), + Sz = Mod:'#info-'(RecName, size), + erlang:make_tuple(Sz, [], [{1, RecName}]); + +newrec(record, Mod, Name, _) -> + newrec(Mod, Name); + +newrec(_, _, _, _) -> + #{}. + +%% newrec/2 + newrec(Mod, Name) -> Mod:'#new-'(Mod:name2rec(Name)). + +%% reformat/5 + +reformat(Name, Map, _Strict, Mod, list) -> + [{F,V} || {F,_} <- Mod:avp_arity(Name), #{F := V} <- [Map]]; + +reformat(Name, Map, Strict, Mod, record_from_map) -> + RecName = Mod:name2rec(Name), + list_to_tuple([RecName | [mget(F, Map, def(A, Strict)) + || {F,A} <- Mod:avp_arity(Name)]]); + +reformat(_, Rec, _, _, _) -> + Rec. + +%% def/2 + +def(1, decode) -> + undefined; + +def(_, _) -> + []. diff --git a/lib/diameter/src/base/diameter_lib.erl b/lib/diameter/src/base/diameter_lib.erl index 8792e97621..1c1ea42cb5 100644 --- a/lib/diameter/src/base/diameter_lib.erl +++ b/lib/diameter/src/base/diameter_lib.erl @@ -283,7 +283,7 @@ ip(T) %% Or not: convert from '.'/':'-separated decimal/hex. ip(Addr) -> - {ok, A} = inet_parse:address(Addr), %% documented in inet(3) + {ok, A} = inet:parse_address(Addr), A. %% --------------------------------------------------------------------------- diff --git a/lib/diameter/src/base/diameter_peer.erl b/lib/diameter/src/base/diameter_peer.erl index 2759f17e64..4cb5a57a54 100644 --- a/lib/diameter/src/base/diameter_peer.erl +++ b/lib/diameter/src/base/diameter_peer.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2015. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -202,10 +202,10 @@ match1(Addr, Match) -> match(Addr, {ok, A}, _) -> Addr == A; match(Addr, {error, _}, RE) -> - match == re:run(inet_parse:ntoa(Addr), RE, [{capture, none}]). + match == re:run(inet:ntoa(Addr), RE, [{capture, none}, caseless]). addr([_|_] = A) -> - inet_parse:address(A); + inet:parse_address(A); addr(A) -> {ok, A}. diff --git a/lib/diameter/src/base/diameter_peer_fsm.erl b/lib/diameter/src/base/diameter_peer_fsm.erl index 1b0dc417e5..d99f11a697 100644 --- a/lib/diameter/src/base/diameter_peer_fsm.erl +++ b/lib/diameter/src/base/diameter_peer_fsm.erl @@ -128,7 +128,8 @@ %% outgoing DPR; boolean says whether or not %% the request was sent explicitly with %% diameter:call/4. - codec :: #{string_decode := boolean(), + codec :: #{decode_format := diameter:decode_format(), + string_decode := boolean(), strict_mbit := boolean(), rfc := 3588 | 6733, ordered_encode := false}, @@ -237,7 +238,7 @@ i({Ack, WPid, {M, Ref} = T, Opts, {SvcOpts, Nodes, Dict0, Svc}}) -> proplists:get_value(dpa_timeout, Opts, ?DPA_TIMEOUT)}), Tmo = proplists:get_value(capx_timeout, Opts, ?CAPX_TIMEOUT), - Strictness = proplists:get_value(capx_strictness, Opts, true), + Strict = proplists:get_value(strict_capx, Opts, true), LengthErr = proplists:get_value(length_errors, Opts, exit), {TPid, Addrs} = start_transport(T, Rest, Svc), @@ -251,9 +252,10 @@ i({Ack, WPid, {M, Ref} = T, Opts, {SvcOpts, Nodes, Dict0, Svc}}) -> mode = M, service = svc(Svc, Addrs), length_errors = LengthErr, - strict = Strictness, + strict = Strict, incoming_maxlen = Maxlen, - codec = maps:with([string_decode, + codec = maps:with([decode_format, + string_decode, strict_mbit, rfc, ordered_encode], @@ -542,11 +544,11 @@ put_route(Pid) -> MRef = monitor(process, Pid), put(Pid, MRef). -%% get_route/2 +%% get_route/3 -%% incoming answer -get_route(_, #diameter_packet{header = #diameter_header{is_request = false}} - = Pkt) -> +%% Incoming answer. +get_route(_, _, #diameter_packet{header = #diameter_header{is_request = false}} + = Pkt) -> Seqs = diameter_codec:sequence_numbers(Pkt), case erase(Seqs) of {Pid, Ref, MRef} -> @@ -557,8 +559,14 @@ get_route(_, #diameter_packet{header = #diameter_header{is_request = false}} false end; -%% incoming request -get_route(Ack, _) -> +%% Requests answered here ... +get_route(_, N, _) + when N == 'CER'; + N == 'DPR' -> + false; + +%% ... or not. +get_route(Ack, _, _) -> Ack. %% erase_route/1 @@ -649,10 +657,6 @@ encode(Rec, Opts, Dict) -> %% incoming/2 -incoming({recv = T, Name, Pkt}, #state{parent = Pid, ack = Ack} = S) -> - Pid ! {T, self(), get_route(Ack, Pkt), Name, Pkt}, - rcv(Name, Pkt, S); - incoming(#diameter_header{is_request = R}, #state{transport = TPid, ack = Ack}) -> R andalso Ack andalso send(TPid, false), @@ -670,98 +674,97 @@ incoming(T, _) -> %% recv/2 -recv(#diameter_packet{header = #diameter_header{} = Hdr} - = Pkt, - #state{dictionary = Dict0} - = S) -> - recv1(diameter_codec:msg_name(Dict0, Hdr), Pkt, S); - -recv(#diameter_packet{header = undefined, - bin = Bin} - = Pkt, - S) -> - recv(diameter_codec:decode_header(Bin), Pkt, S); +recv(#diameter_packet{bin = Bin} = Pkt, S) -> + recv(Bin, Pkt, S); recv(Bin, S) -> - recv(#diameter_packet{bin = Bin}, S). + recv(Bin, Bin, S). + +%% recv/3 + +recv(Bin, Msg, S) -> + recv(diameter_codec:decode_header(Bin), Bin, Msg, S). -%% recv1/3 +%% recv/4 -recv1(_, - #diameter_packet{header = H, bin = Bin}, - #state{incoming_maxlen = M}) +recv(false, Bin, _, #state{length_errors = E}) -> + invalid(E, truncated_header, Bin), + Bin; + +recv(#diameter_header{length = Len} = H, Bin, Msg, #state{length_errors = E, + incoming_maxlen = M, + dictionary = Dict0} + = S) + when E == handle; + 0 == Len rem 4, bit_size(Bin) == 8*Len, size(Bin) =< M -> + recv1(diameter_codec:msg_name(Dict0, H), H, Msg, S); + +recv(H, Bin, _, #state{incoming_maxlen = M}) when M < size(Bin) -> invalid(false, incoming_maxlen_exceeded, {size(Bin), H}), H; +recv(H, Bin, _, #state{length_errors = E}) -> + T = {size(Bin), bit_size(Bin) rem 8, H}, + invalid(E, message_length_mismatch, T), + H. + +%% recv1/4 + %% Ignore anything but an expected CER/CEA if so configured. This is %% non-standard behaviour. -recv1(Name, #diameter_packet{header = H}, #state{state = {'Wait-CEA', _, _}, - strict = false}) +recv1(Name, H, _, #state{state = {'Wait-CEA', _, _}, + strict = false}) when Name /= 'CEA' -> H; -recv1(Name, #diameter_packet{header = H}, #state{state = recv_CER, - strict = false}) +recv1(Name, H, _, #state{state = recv_CER, + strict = false}) when Name /= 'CER' -> H; %% Incoming request after outgoing DPR: discard. Don't discard DPR, so %% both ends don't do so when sending simultaneously. -recv1(Name, - #diameter_packet{header = #diameter_header{is_request = true} = H}, - #state{dpr = {_,_,_}}) +recv1(Name, #diameter_header{is_request = true} = H, _, #state{dpr = {_,_,_}}) when Name /= 'DPR' -> invalid(false, recv_after_outgoing_dpr, H), H; %% Incoming request after incoming DPR: discard. -recv1(_, - #diameter_packet{header = #diameter_header{is_request = true} = H}, - #state{dpr = true}) -> +recv1(_, #diameter_header{is_request = true} = H, _, #state{dpr = true}) -> invalid(false, recv_after_incoming_dpr, H), H; %% DPA with identifier mismatch, or in response to a DPR initiated by %% the service. -recv1('DPA' = N, - #diameter_packet{header = #diameter_header{hop_by_hop_id = Hid, - end_to_end_id = Eid}} - = Pkt, - #state{dpr = {X,H,E}} +recv1('DPA' = Name, + #diameter_header{hop_by_hop_id = Hid, end_to_end_id = Eid} + = H, + Msg, + #state{dpr = {X,HI,EI}} = S) - when H /= Hid; - E /= Eid; + when HI /= Hid; + EI /= Eid; not X -> - rcv(N, Pkt, S); + Pkt = pkt(H, Msg), + handle(Name, Pkt, S); -%% Any other message with a header and no length errors: send to the -%% parent. -recv1(Name, Pkt, #state{}) -> - {recv, Name, Pkt}. +%% Any other message with a header and no length errors. +recv1(Name, H, Msg, #state{parent = Pid, ack = Ack} = S) -> + Pkt = pkt(H, Msg), + Pid ! {recv, self(), get_route(Ack, Name, Pkt), Name, Pkt}, + handle(Name, Pkt, S). -%% recv/3 +%% pkt/2 -recv(#diameter_header{length = Len} - = H, - #diameter_packet{bin = Bin} - = Pkt, - #state{length_errors = E} - = S) - when E == handle; - 0 == Len rem 4, bit_size(Bin) == 8*Len -> - recv(Pkt#diameter_packet{header = H}, S); +pkt(H, Bin) + when is_binary(Bin) -> + #diameter_packet{header = H, + bin = Bin}; -recv(#diameter_header{} - = H, - #diameter_packet{bin = Bin}, - #state{length_errors = E}) -> - T = {size(Bin), bit_size(Bin) rem 8, H}, - invalid(E, message_length_mismatch, T), - Bin; +pkt(H, Pkt) -> + Pkt#diameter_packet{header = H}. -recv(false, #diameter_packet{bin = Bin}, #state{length_errors = E}) -> - invalid(E, truncated_header, Bin), - Bin. +%% invalid/3 %% Note that counters here only count discarded messages. invalid(E, Reason, T) -> @@ -770,39 +773,39 @@ invalid(E, Reason, T) -> ?LOG(Reason, T), ok. -%% rcv/3 +%% handle/3 %% Incoming CEA. -rcv('CEA' = N, - #diameter_packet{header = #diameter_header{end_to_end_id = Eid, - hop_by_hop_id = Hid}} - = Pkt, - #state{state = {'Wait-CEA', Hid, Eid}} - = S) -> +handle('CEA' = N, + #diameter_packet{header = #diameter_header{end_to_end_id = Eid, + hop_by_hop_id = Hid}} + = Pkt, + #state{state = {'Wait-CEA', Hid, Eid}} + = S) -> ?LOG(recv, N), handle_CEA(Pkt, S); %% Incoming CER -rcv('CER' = N, Pkt, #state{state = recv_CER} = S) -> +handle('CER' = N, Pkt, #state{state = recv_CER} = S) -> handle_request(N, Pkt, S); %% Anything but CER/CEA in a non-Open state is an error, as is %% CER/CEA in anything but recv_CER/Wait-CEA. -rcv(Name, _, #state{state = PS}) +handle(Name, _, #state{state = PS}) when PS /= 'Open'; Name == 'CER'; Name == 'CEA' -> {stop, {Name, PS}}; -rcv('DPR' = N, Pkt, S) -> +handle('DPR' = N, Pkt, S) -> handle_request(N, Pkt, S); %% DPA in response to DPR, with the expected identifiers. -rcv('DPA' = N, - #diameter_packet{header = #diameter_header{end_to_end_id = Eid, - hop_by_hop_id = Hid} - = H} - = Pkt, +handle('DPA' = N, + #diameter_packet{header = #diameter_header{end_to_end_id = Eid, + hop_by_hop_id = Hid} + = H} + = Pkt, #state{dictionary = Dict0, transport = TPid, dpr = {X, Hid, Eid}, @@ -813,7 +816,8 @@ rcv('DPA' = N, %% service: explicit DPR is counted in the same way %% as other explicitly sent requests. incr(recv, H, Dict0), - incr_rc(recv, diameter_codec:decode(Dict0, Opts, Pkt), Dict0) + {_, RecPkt} = decode(Dict0, Opts, Pkt), + incr_rc(recv, RecPkt, Dict0) end, diameter_peer:close(TPid), {stop, N}; @@ -821,13 +825,13 @@ rcv('DPA' = N, %% Ignore an unsolicited DPA in particular. Note that dpa_timeout %% deals with the case in which the peer sends the wrong identifiers %% in DPA. -rcv('DPA' = N, #diameter_packet{header = H}, _) -> +handle('DPA' = N, #diameter_packet{header = H}, _) -> ?LOG(ignored, N), %% Note that these aren't counted in the normal recv counter. diameter_stats:incr({diameter_codec:msg_id(H), recv, ignored}), ok; -rcv(_, _, _) -> +handle(_, _, _) -> ok. %% incr/3 @@ -917,21 +921,30 @@ handle_request(Name, = S) -> ?LOG(recv, Name), incr(recv, H, Dict0), - send_answer(Name, diameter_codec:decode(Dict0, Opts, Pkt), S). + send_answer(Name, decode(Dict0, Opts, Pkt), S). + +%% decode/3 +%% +%% Decode the message as record for diameter_capx, and in the +%% configured format for events. + +decode(Dict0, Opts, Pkt) -> + {diameter_codec:decode(Dict0, Opts, Pkt), + diameter_codec:decode(Dict0, Opts#{decode_format := record}, Pkt)}. %% send_answer/3 -send_answer(Type, ReqPkt, #state{transport = TPid, - dictionary = Dict, - codec = Opts} - = S) -> - incr_error(recv, ReqPkt, Dict), +send_answer(Type, {DecPkt, RecPkt}, #state{transport = TPid, + dictionary = Dict, + codec = Opts} + = S) -> + incr_error(recv, RecPkt, Dict), #diameter_packet{header = H, transport_data = TD} - = ReqPkt, + = RecPkt, - {Msg, PostF} = build_answer(Type, ReqPkt, S), + {Msg, PostF} = build_answer(Type, DecPkt, RecPkt, S), %% An answer message clears the R and T flags and retains the P %% flag. The E flag is set at encode. @@ -959,15 +972,15 @@ eval([F|A], S) -> eval(T, _) -> close(T). -%% build_answer/3 +%% build_answer/4 build_answer('CER', + DecPkt, #diameter_packet{msg = CER, header = #diameter_header{version = ?DIAMETER_VERSION, is_error = false}, - errors = []} - = Pkt, + errors = []}, #state{dictionary = Dict0} = S) -> {SupportedApps, RCaps, CEA} = recv_CER(CER, S), @@ -985,25 +998,25 @@ build_answer('CER', orelse ?THROW(4003), %% DIAMETER_ELECTION_LOST caps_cb(Caps) of - N -> {cea(CEA, N, Dict0), [fun open/5, Pkt, + N -> {cea(CEA, N, Dict0), [fun open/5, DecPkt, SupportedApps, Caps, {accept, inband_security(IS)}]} catch ?FAILURE(Reason) -> - rejected(Reason, {'CER', Reason, Caps, Pkt}, S) + rejected(Reason, {'CER', Reason, Caps, DecPkt}, S) end; %% The error checks below are similar to those in diameter_traffic for %% other messages. Should factor out the commonality. build_answer(Type, + DecPkt, #diameter_packet{header = H, - errors = Es} - = Pkt, + errors = Es}, S) -> {RC, FailedAVP} = result_code(Type, H, Es), - {answer(Type, RC, FailedAVP, S), post(Type, RC, Pkt, S)}. + {answer(Type, RC, FailedAVP, S), post(Type, RC, DecPkt, S)}. inband_security([]) -> ?NO_INBAND_SECURITY; @@ -1175,12 +1188,10 @@ handle_CEA(#diameter_packet{header = H} = S) -> incr(recv, H, Dict0), - #diameter_packet{} - = DPkt - = diameter_codec:decode(Dict0, Opts, Pkt), + {DecPkt, RecPkt} = decode(Dict0, Opts, Pkt), - RC = result_code(incr_rc(recv, DPkt, Dict0)), - {SApps, IS, RCaps} = recv_CEA(DPkt, S), + RC = result_code(incr_rc(recv, RecPkt, Dict0)), + {SApps, IS, RCaps} = recv_CEA(RecPkt, S), #diameter_caps{origin_host = {OH, DH}} = Caps @@ -1203,9 +1214,9 @@ handle_CEA(#diameter_packet{header = H} orelse ?THROW(election_lost), caps_cb(Caps) of - _ -> open(DPkt, SApps, Caps, {connect, hd([_] = IS)}, S) + _ -> open(DecPkt, SApps, Caps, {connect, hd([_] = IS)}, S) catch - ?FAILURE(Reason) -> close({'CEA', Reason, Caps, DPkt}) + ?FAILURE(Reason) -> close({'CEA', Reason, Caps, DecPkt}) end. %% Check more than the result code since the peer could send success %% regardless. If not 2001 then a peer_up callback could do anything diff --git a/lib/diameter/src/base/diameter_service.erl b/lib/diameter/src/base/diameter_service.erl index a976a8b998..31dd92f878 100644 --- a/lib/diameter/src/base/diameter_service.erl +++ b/lib/diameter/src/base/diameter_service.erl @@ -112,8 +112,24 @@ use_shared_peers := diameter:remotes(),%% use from restrict_connections := diameter:restriction(), incoming_maxlen := diameter:message_length(), + strict_arities => diameter:strict_arities(), strict_mbit := boolean(), + decode_format := diameter:decode_format(), + avp_dictionaries => nonempty_list(module()), + traffic_counters := boolean(), string_decode := boolean(), + capabilities_cb => diameter:evaluable(), + pool_size => pos_integer(), + capx_timeout => diameter:'Unsigned32'(), + strict_capx => boolean(), + disconnect_cb => diameter:evaluable(), + dpr_timeout => diameter:'Unsigned32'(), + dpa_timeout => diameter:'Unsigned32'(), + length_errors => exit | handle | discard, + connect_timer => diameter:'Unsigned32'(), + watchdog_timer => diameter:'Unsigned32'() + | {module(), atom(), list()}, + watchdog_config => [{okay|suspect, non_neg_integer()}], spawn_opt := list() | {module(), atom(), list()}}}). %% Record representing an RFC 3539 watchdog process implemented by @@ -514,6 +530,13 @@ transition({tc_timeout, T}, S) -> tc_timeout(T, S), ok; +transition({nodeup, Node, _}, S) -> + nodeup(Node, S), + ok; + +transition({nodedown, _Node, _}, _) -> + ok; + transition(Req, S) -> unexpected(handle_info, [Req], S), ok. @@ -679,12 +702,15 @@ i(SvcName) -> cfg_acc({SvcName, #diameter_service{applications = Apps} = Rec, Opts}, {false, Acc}) -> lists:foreach(fun init_mod/1, Apps), + #{monitor := M} + = SvcOpts + = service_opts(Opts), S = #state{service_name = SvcName, service = Rec#diameter_service{pid = self()}, local = init_peers(), remote = init_peers(), - monitor = mref(get_value(monitor, Opts)), - options = service_options(lists:keydelete(monitor, 1, Opts))}, + monitor = mref(M), + options = maps:remove(monitor, SvcOpts)}, {S, Acc}; cfg_acc({_Ref, Type, _Opts} = T, {S, Acc}) @@ -699,8 +725,29 @@ init_peers() -> %% Alias, %% TPid} -service_options(Opts) -> - maps:from_list(Opts). +service_opts(Opts) -> + remove([{strict_arities, true}, + {avp_dictionaries, []}], + maps:merge(maps:from_list([{monitor, false} | def_opts()]), + maps:from_list(Opts))). + +remove(List, Map) -> + maps:filter(fun(K,V) -> not lists:member({K,V}, List) end, + Map). + +def_opts() -> %% defaults on the service map + [{share_peers, false}, + {use_shared_peers, false}, + {sequence, {0,32}}, + {restrict_connections, nodes}, + {incoming_maxlen, 16#FFFFFF}, + {strict_arities, true}, + {strict_mbit, true}, + {decode_format, record}, + {avp_dictionaries, []}, + {traffic_counters, true}, + {string_decode, true}, + {spawn_opt, []}]. mref(false = No) -> No; @@ -709,6 +756,8 @@ mref(P) -> init_shared(#state{options = #{use_shared_peers := T}, service_name = Svc}) -> + T == false orelse net_kernel:monitor_nodes(true, [{node_type, visible}, + nodedown_reason]), notify(T, Svc, {service, self()}). init_mod(#diameter_app{alias = Alias, @@ -718,16 +767,17 @@ init_mod(#diameter_app{alias = Alias, start_fsm({Ref, Type, Opts}, S) -> start(Ref, {Type, Opts}, S). -get_value(Key, Vs) -> - {_, V} = lists:keyfind(Key, 1, Vs), - V. - notify(Share, SvcName, T) -> Nodes = remotes(Share), [] /= Nodes andalso diameter_peer:notify(Nodes, SvcName, T). %% Test for the empty list for upgrade reasons: there's no %% diameter_peer:notify/3 in old code. +nodeup(Node, #state{options = #{share_peers := SP}, + service_name = SvcName}) -> + lists:member(Node, remotes(SP)) + andalso diameter_peer:notify([Node], SvcName, {service, self()}). + remotes(false) -> []; @@ -806,7 +856,7 @@ start(Ref, Type, Opts, State) -> start(Ref, Type, Opts, N, #state{watchdogT = WatchdogT, local = {PeerT, _, _}, options = #{string_decode := SD} - = SvcOpts0, + = SvcOpts, service_name = SvcName, service = Svc0}) when Type == connect; @@ -815,12 +865,12 @@ start(Ref, Type, Opts, N, #state{watchdogT = WatchdogT, = Svc1 = merge_service(Opts, Svc0), Svc = binary_caps(Svc1, SD), - SvcOpts = merge_options(Opts, SvcOpts0), - RecvData = diameter_traffic:make_recvdata([SvcName, PeerT, Apps, SvcOpts]), - T = {Opts, SvcOpts, RecvData, Svc}, + {SOpts, TOpts} = merge_opts(SvcOpts, Opts), + RecvData = diameter_traffic:make_recvdata([SvcName, PeerT, Apps, SOpts]), + T = {TOpts, SOpts, RecvData, Svc}, Rec = #watchdog{type = Type, ref = Ref, - options = Opts}, + options = TOpts}, diameter_lib:fold_n(fun(_,A) -> [wd(Type, Ref, T, WatchdogT, Rec) | A] @@ -828,10 +878,14 @@ start(Ref, Type, Opts, N, #state{watchdogT = WatchdogT, [], N). -merge_options(Opts, SvcOpts) -> - Keys = maps:keys(SvcOpts), - Map = maps:from_list([KV || {K,_} = KV <- Opts, lists:member(K, Keys)]), - maps:merge(SvcOpts, Map). +merge_opts(SvcOpts, Opts) -> + Keys = [K || {K,_} <- def_opts()], + SO = [T || {K,_} = T <- Opts, lists:member(K, Keys)], + TO = Opts -- SO, + {maps:merge(maps:with(Keys, SvcOpts), maps:from_list(SO)), + TO ++ [T || {K,_} = T <- maps:to_list(SvcOpts), + not lists:member(K, Keys), + not lists:keymember(K, 1, Opts)]}. binary_caps(Svc, true) -> Svc; @@ -1400,9 +1454,15 @@ is_remote(Pid, T) -> %% # remote_peer_up/4 %% --------------------------------------------------------------------------- -remote_peer_up(TPid, Aliases, Caps, #state{options = #{use_shared_peers := T}} +remote_peer_up(TPid, Aliases, Caps, #state{options = #{use_shared_peers := T}, + remote = {PeerT, _, _}} = S) -> - is_remote(TPid, T) andalso rpu(TPid, Aliases, Caps, S). + is_remote(TPid, T) + andalso not ets:member(PeerT, TPid) + andalso rpu(TPid, Aliases, Caps, S). + +%% Notification can be duplicate since remote nodes push and the local +%% node pulls. rpu(TPid, Aliases, Caps, #state{service = Svc, remote = RT}) -> #diameter_service{applications = Apps} = Svc, @@ -1412,6 +1472,7 @@ rpu(TPid, Aliases, Caps, #state{service = Svc, remote = RT}) -> rpu(_, [] = No, _, _) -> No; + rpu(TPid, Aliases, Caps, {PeerT, _, _} = RT) -> monitor(process, TPid), ets:insert(PeerT, #peer{pid = TPid, diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index 85378babea..f510f40a17 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -70,13 +70,17 @@ timeout = 5000 :: 0..16#FFFFFFFF, %% for outgoing requests detach = false :: boolean()}). -%% Term passed back to receive_message/6 with every incoming message. +%% Term passed back to receive_message/5 with every incoming message. -record(recvdata, {peerT :: ets:tid(), service_name :: diameter:service_name(), apps :: [#diameter_app{}], sequence :: diameter:sequence(), - codec :: #{string_decode := boolean(), + counters :: boolean(), + codec :: #{decode_format := diameter:decode_format(), + avp_dictionaries => nonempty_list(module()), + string_decode := boolean(), + strict_arities => diameter:strict_arities(), strict_mbit := boolean(), incoming_maxlen := diameter:message_length()}}). %% Note that incoming_maxlen is currently handled in diameter_peer_fsm, @@ -89,6 +93,7 @@ caller :: pid() | undefined, %% calling process handler :: pid(), %% request process peer :: undefined | {pid(), #diameter_caps{}}, + caps :: undefined, %% no longer used packet :: #diameter_packet{} | undefined}). %% of request %% --------------------------------------------------------------------------- @@ -96,13 +101,17 @@ %% --------------------------------------------------------------------------- make_recvdata([SvcName, PeerT, Apps, SvcOpts | _]) -> - #{sequence := {_,_} = Mask, spawn_opt := Opts} + #{sequence := {_,_} = Mask, spawn_opt := Opts, traffic_counters := B} = SvcOpts, {Opts, #recvdata{service_name = SvcName, peerT = PeerT, apps = Apps, sequence = Mask, - codec = maps:with([string_decode, + counters = B, + codec = maps:with([decode_format, + avp_dictionaries, + string_decode, + strict_arities, strict_mbit, ordered_encode, incoming_maxlen], @@ -182,7 +191,7 @@ incr_error(Dir, Id, TPid) -> %% --------------------------------------------------------------------------- -spec incr_rc(send|recv, Pkt, TPid, DictT) - -> {Counter, non_neg_integer()} + -> Counter | Reason when Pkt :: #diameter_packet{}, TPid :: pid(), @@ -193,18 +202,26 @@ incr_error(Dir, Id, TPid) -> | {'Experimental-Result', integer(), integer()}, Reason :: atom(). -incr_rc(Dir, Pkt, TPid, {_, AppDict, _} = DictT) -> - try - incr_result(Dir, Pkt, TPid, DictT) +incr_rc(Dir, Pkt, TPid, {MsgDict, AppDict, Dict0}) -> + incr_rc(Dir, Pkt, TPid, MsgDict, AppDict, Dict0); + +incr_rc(Dir, Pkt, TPid, Dict0) -> + incr_rc(Dir, Pkt, TPid, Dict0, Dict0, Dict0). + +%% incr_rc/6 + +incr_rc(Dir, Pkt, TPid, MsgDict, AppDict, Dict0) -> + try get_result(Dir, MsgDict, Dict0, Pkt) of + false -> + unknown; + Avp -> + incr_result(Dir, Avp, Pkt, TPid, AppDict) catch exit: {E,_} when E == no_result_code; E == invalid_error_bit -> incr(TPid, {msg_id(Pkt#diameter_packet.header, AppDict), Dir, E}), E - end; - -incr_rc(Dir, Pkt, TPid, Dict0) -> - incr_rc(Dir, Pkt, TPid, {Dict0, Dict0, Dict0}). + end. %% --------------------------------------------------------------------------- %% receive_message/5 @@ -216,13 +233,13 @@ incr_rc(Dir, Pkt, TPid, Dict0) -> -> pid() %% request handler | boolean() %% answer, known request or not | discard %% request discarded by MFA - when Route :: {Handler, RequestRef, Seqs} + when Route :: {Handler, RequestRef, TPid} | Ack, RecvData :: {[SpawnOpt], #recvdata{}}, SpawnOpt :: term(), Handler :: pid(), RequestRef :: reference(), - Seqs :: {0..16#FFFFFFFF, 0..16#FFFFFFFF}, + TPid :: pid(), Ack :: boolean(). receive_message(TPid, Route, Pkt, Dict0, RecvData) -> @@ -303,14 +320,15 @@ recv_request(Ack, = Pkt, Dict0, #recvdata{peerT = PeerT, - apps = Apps} + apps = Apps, + counters = Count} = RecvData) -> Ack andalso (TPid ! {handler, self()}), case diameter_service:find_incoming_app(PeerT, TPid, Id, Apps) of {#diameter_app{id = Aid, dictionary = AppDict} = App, Caps} -> - incr(recv, Pkt, TPid, AppDict), + Count andalso incr(recv, Pkt, TPid, AppDict), DecPkt = decode(Aid, AppDict, RecvData, Pkt), - incr_error(recv, DecPkt, TPid, AppDict), + Count andalso incr_error(recv, DecPkt, TPid, AppDict), send_A(recv_R(App, TPid, Dict0, Caps, RecvData, DecPkt), TPid, App, @@ -323,9 +341,7 @@ recv_request(Ack, %% A request was sent for an application that is not %% supported. RC = 3007, - Es = Pkt#diameter_packet.errors, - DecPkt = Pkt#diameter_packet{avps = collect_avps(Pkt), - errors = [RC | Es]}, + DecPkt = diameter_codec:collect_avps(Pkt), send_answer(answer_message(RC, Dict0, Caps, DecPkt), TPid, Dict0, @@ -338,17 +354,11 @@ recv_request(Ack, No end. +%% decode/4 + decode(Id, Dict, #recvdata{codec = Opts}, Pkt) -> errors(Id, diameter_codec:decode(Id, Dict, Opts, Pkt)). -collect_avps(Pkt) -> - case diameter_codec:collect_avps(Pkt) of - {_Error, Avps} -> - Avps; - Avps -> - Avps - end. - %% send_A/7 send_A([T | Fs], TPid, App, Dict0, RecvData, DecPkt, Caps) -> @@ -541,6 +551,7 @@ send_A({call, Opts}, TPid, App, Dict0, RecvData, Pkt, Caps, Fs) -> MsgDict, AppDict, Dict0, + RecvData#recvdata.counters, Fs); RC -> send_answer(answer_message(RC, Dict0, Caps, Pkt), @@ -584,14 +595,22 @@ send_answer(Ans, TPid, MsgDict, AppDict, Dict0, RecvData, DecPkt, Fs) -> TPid, RecvData#recvdata.codec, make_answer_packet(Ans, DecPkt, MsgDict, Dict0)), - send_answer(Pkt, TPid, MsgDict, AppDict, Dict0, Fs). + send_answer(Pkt, + TPid, + MsgDict, + AppDict, + Dict0, + RecvData#recvdata.counters, + Fs). -%% send_answer/6 +%% send_answer/7 -send_answer(Pkt, TPid, MsgDict, AppDict, Dict0, [EvalPktFs | EvalFs]) -> +send_answer(Pkt, TPid, MsgDict, AppDict, Dict0, Count, [EvalPktFs | EvalFs]) -> eval_packet(Pkt, EvalPktFs), - incr(send, Pkt, TPid, AppDict), - incr_rc(send, Pkt, TPid, {MsgDict, AppDict, Dict0}), %% count outgoing + Count andalso begin + incr(send, Pkt, TPid, AppDict), + incr_rc(send, Pkt, TPid, MsgDict, AppDict, Dict0) + end, send(TPid, z(Pkt), _Route = self()), lists:foreach(fun diameter_lib:eval/1, EvalFs). @@ -619,7 +638,7 @@ is_answer_message(#diameter_packet{msg = Msg}, Dict0) -> is_answer_message([#diameter_header{is_request = R, is_error = E} | _], _) -> E andalso not R; -%% Message sent as a tagged avp/value list. +%% Message sent as a map or tagged avp/value list. is_answer_message([Name | _], _) -> Name == 'answer-message'; @@ -867,7 +886,10 @@ reset(Msg, [RC | Avps], Dict) -> %% set/3 -%% Reply as name and tuple list ... +%% Reply as name/values list ... +set([Name|As], Avps, _) + when is_map(As) -> + [Name | maps:merge(As, maps:from_list(Avps))]; set([_|_] = Ans, Avps, _) -> Ans ++ Avps; %% Values nearer tail take precedence. @@ -900,33 +922,44 @@ failed_avp(_, [] = No, _) -> failed_avp(Msg, [_|_] = Avps, Dict) -> [failed(Msg, [{'AVP', Avps}], Dict)]. -%% Reply as name and tuple list ... -failed([MsgName | Values], FailedAvp, Dict) -> - RecName = Dict:msg2rec(MsgName), +%% failed/3 + +failed(Msg, FailedAvp, Dict) -> + RecName = msg2rec(Msg, Dict), try - Dict:'#info-'(RecName, {index, 'Failed-AVP'}), + Dict:'#info-'(RecName, {index, 'Failed-AVP'}), %% assert existence {'Failed-AVP', [FailedAvp]} catch error: _ -> - Avps = proplists:get_value('AVP', Values, []), + Avps = values(Msg, 'AVP', Dict), A = #diameter_avp{name = 'Failed-AVP', value = FailedAvp}, {'AVP', [A|Avps]} + end. + +%% msg2rec/2 + +%% Message as name/values list ... +msg2rec([MsgName | _], Dict) -> + Dict:msg2rec(MsgName); + +%% ... or record. +msg2rec(Rec, _) -> + element(1, Rec). + +%% values/2 + +%% Message as name/values list ... +values([_ | Avps], F, _) -> + if is_map(Avps) -> + maps:get(F, Avps, []); + is_list(Avps) -> + proplists:get_value(F, Avps, []) end; %% ... or record. -failed(Rec, FailedAvp, Dict) -> - try - RecName = element(1, Rec), - Dict:'#info-'(RecName, {index, 'Failed-AVP'}), - {'Failed-AVP', [FailedAvp]} - catch - error: _ -> - Avps = Dict:'#get-'('AVP', Rec), - A = #diameter_avp{name = 'Failed-AVP', - value = FailedAvp}, - {'AVP', [A|Avps]} - end. +values(Rec, F, Dict) -> + Dict:'#get-'(F, Rec). %% 3. Diameter Header %% @@ -1003,15 +1036,15 @@ answer_message(RC, origin_realm = {OR,_}}, #diameter_packet{avps = Avps, errors = Es}) -> - {Code, _, Vid} = Dict0:avp_header('Session-Id'), ['answer-message', {'Origin-Host', OH}, {'Origin-Realm', OR}, - {'Result-Code', RC}] - ++ session_id(Code, Vid, Avps) - ++ failed_avp(RC, Es). + {'Result-Code', RC} + | session_id(Dict0, Avps) + ++ failed_avp(RC, Es) + ++ proxy_info(Dict0, Avps)]. -session_id(Code, Vid, Avps) - when is_list(Avps) -> +session_id(Dict0, Avps) -> + {Code, _, Vid} = Dict0:avp_header('Session-Id'), try #diameter_avp{data = Bin} = find_avp(Code, Vid, Avps), [{'Session-Id', [Bin]}] @@ -1029,6 +1062,14 @@ failed_avp(RC, [_ | Es]) -> failed_avp(_, [] = No) -> No. +proxy_info(Dict0, Avps) -> + {Code, _, Vid} = Dict0:avp_header('Proxy-Info'), + [{'AVP', [A#diameter_avp{value = undefined} + || [#diameter_avp{code = C, vendor_id = I} = A | _] + <- Avps, + C == Code, + I == Vid]}]. + %% find_avp/3 %% Grouped ... @@ -1102,48 +1143,31 @@ find_avp(Code, VId, [_ | Avps]) -> %% Message sent as a header/avps list. incr_result(send = Dir, - #diameter_packet{msg = [#diameter_header{} = H | _]} - = Pkt, + Avp, + #diameter_packet{msg = [#diameter_header{} = H | _]}, TPid, - DictT) -> - incr_res(Dir, Pkt#diameter_packet{header = H}, TPid, DictT); - -%% Outgoing message as binary: don't count. (Sending binaries is only -%% partially supported.) -incr_result(send, #diameter_packet{header = undefined = No}, _, _) -> - No; + AppDict) -> + incr_result(Dir, Avp, H, [], TPid, AppDict); %% Incoming or outgoing. Outgoing with encode errors never gets here %% since encode fails. -incr_result(Dir, Pkt, TPid, DictT) -> - incr_res(Dir, Pkt, TPid, DictT). - -incr_res(Dir, - #diameter_packet{header = #diameter_header{is_error = E} - = Hdr, - errors = Es} - = Pkt, - TPid, - DictT) -> - {MsgDict, AppDict, Dict0} = DictT, +incr_result(Dir, Avp, Pkt, TPid, AppDict) -> + #diameter_packet{header = H, errors = Es} + = Pkt, + incr_result(Dir, Avp, H, Es, TPid, AppDict). + +%% incr_result/6 +incr_result(Dir, Avp, Hdr, Es, TPid, AppDict) -> Id = msg_id(Hdr, AppDict), %% Could be {relay, 0}, in which case the R-bit is redundant since %% only answers are being counted. Let it be however, so that the %% same tuple is in both send/recv and result code counters. %% Count incoming decode errors. - recv /= Dir orelse [] == Es orelse incr_error(Dir, Id, TPid, AppDict), - - %% Exit on a missing result code. - T = rc_counter(MsgDict, Dir, Pkt), - T == false andalso ?LOGX(no_result_code, {MsgDict, Dir, Hdr}), - {Ctr, RC, Avp} = T, - - %% Or on an inappropriate value. - is_result(RC, E, Dict0) - orelse ?LOGX(invalid_error_bit, {MsgDict, Dir, Hdr, Avp}), + send == Dir orelse [] == Es orelse incr_error(Dir, Id, TPid, AppDict), + Ctr = rcc(Avp), incr(TPid, {Id, Dir, Ctr}), Ctr. @@ -1188,7 +1212,50 @@ is_result(RC, true, _) -> incr(TPid, Counter) -> diameter_stats:incr(Counter, TPid, 1). -%% rc_counter/3 +%% rcc/1 + +rcc(#diameter_avp{name = 'Result-Code' = Name, value = V}) -> + {Name, head(V)}; + +rcc(#diameter_avp{name = 'Experimental-Result', value = V}) -> + head(V). + +%% head/1 + +head([V|_]) -> + V; +head(V) -> + V. + +%% rcv/1 + +rcv(#diameter_avp{name = N, value = V}) -> + rcv(N, head(V)). + +%% rcv/2 + +rcv('Experimental-Result', {_,_,N}) -> + N; + +rcv('Result-Code', N) -> + N. + +%% get_result/4 + +%% Message sent as binary: no checks or counting. +get_result(_, _, _, #diameter_packet{header = undefined}) -> + false; + +get_result(Dir, MsgDict, Dict0, Pkt) -> + Avp = get_result(MsgDict, msg(Dir, Pkt)), + Hdr = Pkt#diameter_packet.header, + %% Exit on a missing result code or inappropriate value. + Avp == false + andalso ?LOGX(no_result_code, {MsgDict, Dir, Hdr}), + E = Hdr#diameter_header.is_error, + is_result(rcv(Avp), E, Dict0) + orelse ?LOGX(invalid_error_bit, {MsgDict, Dir, Hdr, Avp}), + Avp. %% RFC 3588, 7.6: %% @@ -1196,46 +1263,29 @@ incr(TPid, Counter) -> %% applications MUST include either one Result-Code AVP or one %% Experimental-Result AVP. -rc_counter(Dict, Dir, #diameter_packet{header = H, - avps = As, - msg = Msg}) +%% msg/2 + +msg(Dir, #diameter_packet{header = H, + avps = As, + msg = Msg}) when Dir == recv; %% decoded incoming Msg == undefined -> %% relayed outgoing - rc_counter(Dict, [H|As]); - -rc_counter(Dict, _, #diameter_packet{msg = Msg}) -> - rc_counter(Dict, Msg). - -rc_counter(Dict, Msg) -> - rcc(get_result(Dict, Msg)). - -rcc(#diameter_avp{name = 'Result-Code' = Name, value = N} = A) - when is_integer(N) -> - {{Name, N}, N, A}; - -rcc(#diameter_avp{name = 'Result-Code' = Name, value = [N|_]} = A) - when is_integer(N) -> - {{Name, N}, N, A}; + [H|As]; -rcc(#diameter_avp{name = 'Experimental-Result', value = {_,_,N} = T} = A) - when is_integer(N) -> - {T, N, A}; - -rcc(#diameter_avp{name = 'Experimental-Result', value = [{_,_,N} = T|_]} = A) - when is_integer(N) -> - {T, N, A}; - -rcc(_) -> - false. +msg(_, #diameter_packet{msg = Msg}) -> + Msg. %% get_result/2 get_result(Dict, Msg) -> try [throw(A) || N <- ['Result-Code', 'Experimental-Result'], - #diameter_avp{} = A <- [get_avp(Dict, N, Msg)]] + #diameter_avp{} = A <- [get_avp(Dict, N, Msg)], + is_integer(catch rcv(A))], + false catch - #diameter_avp{} = A -> A + #diameter_avp{} = A -> + A end. x(T) -> @@ -1359,7 +1409,7 @@ make_opts([T | _], _, _, _, _, _) -> send_request({{TPid, _Caps} = TC, App} = Transport, - #{sequence := Mask} + #{sequence := Mask, traffic_counters := Count} = SvcOpts, Msg0, CallOpts, @@ -1375,9 +1425,15 @@ send_request({{TPid, _Caps} = TC, App} SvcOpts, ReqPkt), eval_packet(EncPkt, Fs), - T = send_R(ReqPkt, EncPkt, Transport, CallOpts, Caller, SvcName), + T = send_R(ReqPkt, + EncPkt, + Transport, + CallOpts, + Caller, + Count, + SvcName), Ans = recv_answer(SvcName, App, CallOpts, T), - handle_answer(SvcName, SvcOpts, App, Ans); + handle_answer(SvcName, Count, SvcOpts, App, Ans); {discard, Reason} -> {error, Reason}; discard -> @@ -1520,6 +1576,7 @@ send_R(ReqPkt, {{TPid, _Caps} = TC, #diameter_app{dictionary = AppDict}}, #options{timeout = Timeout}, {Pid, Ref}, + Count, SvcName) -> Req = #request{ref = Ref, caller = Pid, @@ -1527,7 +1584,7 @@ send_R(ReqPkt, peer = TC, packet = ReqPkt}, - incr(send, EncPkt, TPid, AppDict), + Count andalso incr(send, EncPkt, TPid, AppDict), {TRef, MRef} = zend_requezt(TPid, EncPkt, Req, SvcName, Timeout), Pid ! Ref, %% tell caller a send has been attempted {TRef, MRef, Req}. @@ -1559,15 +1616,16 @@ failover(SvcName, App, Req, CallOpts) -> CallOpts, SvcName). -%% handle_answer/4 +%% handle_answer/5 -handle_answer(SvcName, _, App, {error, Req, Reason}) -> +handle_answer(SvcName, _, _, App, {error, Req, Reason}) -> #request{packet = Pkt, peer = {_TPid, _Caps} = TC} = Req, cb(App, handle_error, [Reason, msg(Pkt), SvcName, TC]); handle_answer(SvcName, + Count, SvcOpts, #diameter_app{id = Id, dictionary = AppDict, @@ -1581,43 +1639,50 @@ handle_answer(SvcName, #request{peer = {TPid, _}} = Req, - incr(recv, DecPkt, TPid, AppDict), - - AnsPkt = try - incr_result(recv, DecPkt, TPid, {MsgDict, AppDict, Dict0}) - of - _ -> DecPkt - catch - exit: {no_result_code, _} -> - %% RFC 6733 requires one of Result-Code or - %% Experimental-Result, but the decode will have - %% detected a missing AVP. If both are optional in - %% the dictionary then this isn't a decode error: - %% just continue on. - DecPkt; - exit: {invalid_error_bit, {_, _, _, Avp}} -> - #diameter_packet{errors = Es} - = DecPkt, - E = {5004, Avp}, - DecPkt#diameter_packet{errors = [E|Es]} - end, - - handle_answer(AnsPkt, SvcName, App, AE, Req). + answer(answer(DecPkt, TPid, MsgDict, AppDict, Dict0, Count), + SvcName, + App, + AE, + Req). + +%% answer/6 + +answer(DecPkt, TPid, MsgDict, AppDict, Dict0, Count) -> + Count andalso incr(recv, DecPkt, TPid, AppDict), + try get_result(recv, MsgDict, Dict0, DecPkt) of + Avp -> + Count andalso false /= Avp + andalso incr_result(recv, Avp, DecPkt, TPid, AppDict), + DecPkt + catch + exit: {no_result_code, _} -> + %% RFC 6733 requires one of Result-Code or + %% Experimental-Result, but the decode will have + %% detected a missing AVP. If both are optional in + %% the dictionary then this isn't a decode error: + %% just continue on. + DecPkt; + exit: {invalid_error_bit, {_, _, _, Avp}} -> + #diameter_packet{errors = Es} + = DecPkt, + E = {5004, Avp}, + DecPkt#diameter_packet{errors = [E|Es]} + end. -%% handle_answer/5 +%% answer/5 -handle_answer(#diameter_packet{errors = Es} - = Pkt, - SvcName, - App, - AE, - #request{peer = {_TPid, _Caps} = TC, - packet = P}) +answer(#diameter_packet{errors = Es} + = Pkt, + SvcName, + App, + AE, + #request{peer = {_TPid, _Caps} = TC, + packet = P}) when callback == AE; [] == Es -> cb(App, handle_answer, [Pkt, msg(P), SvcName, TC]); -handle_answer(#diameter_packet{header = H}, SvcName, _, AE, _) -> +answer(#diameter_packet{header = H}, SvcName, _, AE, _) -> handle_error(H, SvcName, AE). %% handle_error/3 @@ -1830,10 +1895,8 @@ get_destination(Dict, Msg) -> [str(get_avp_value(Dict, D, Msg)) || D <- ['Destination-Realm', 'Destination-Host']]. -%% This is not entirely correct. The avp could have an arity 1, in -%% which case an empty list is a DiameterIdentity of length 0 rather -%% than the list of no values we treat it as by mapping to undefined. -%% This behaviour is documented. +%% A DiameterIdentity has length at least one, so an empty list is not +%% a Realm/Host. str([]) -> undefined; str(T) -> @@ -1841,16 +1904,12 @@ str(T) -> %% get_avp/3 %% -%% Find an AVP in a message of one of three forms: -%% -%% - a message record (as generated from a .dia spec) or -%% - a list of an atom message name followed by 2-tuple, avp name/value pairs. -%% - a list of a #diameter_header{} followed by #diameter_avp{} records, -%% -%% In the first two forms a dictionary module is used at encode to -%% identify the type of the AVP and its arity in the message in -%% question. The third form allows messages to be sent as is, without -%% a dictionary, which is needed in the case of relay agents, for one. +%% Find an AVP in a message in one of the decoded formats, or as a +%% header/avps list. There are only four AVPs that are extracted here: +%% Result-Code and Experimental-Result in order when constructing +%% counter keys, and Destination-Host/Realm when selecting a next-hop +%% peer. Experimental-Result is the only of type Grouped, and is given +%% special treatment in order to return the value as a record. %% Messages will be header/avps list as a relay and the only AVP's we %% look for are in the common dictionary. This is required since the @@ -1859,37 +1918,58 @@ str(T) -> get_avp(?RELAY, Name, Msg) -> get_avp(?BASE, Name, Msg); -%% Message as a header/avps list. +%% Message as header/avps list. get_avp(Dict, Name, [#diameter_header{} | Avps]) -> try - {Code, _, VId} = Dict:avp_header(Name), - find_avp(Code, VId, Avps) - of - A -> - (avp_decode(Dict, Name, ungroup(A)))#diameter_avp{name = Name} + {Code, _, Vid} = Dict:avp_header(Name), + A = find_avp(Code, Vid, Avps), + avp_decode(Dict, Name, ungroup(A)) catch error: _ -> undefined end; -%% Outgoing message as a name/values list. +%% Message as name/values list ... get_avp(_, Name, [_MsgName | Avps]) -> - case lists:keyfind(Name, 1, Avps) of + case find(Name, Avps) of {_, V} -> - #diameter_avp{name = Name, value = V}; + #diameter_avp{name = Name, value = value(Name, V)}; _ -> undefined end; -%% Message is typically a record but not necessarily. +%% ... or record. get_avp(Dict, Name, Rec) -> - try - #diameter_avp{name = Name, value = Dict:'#get-'(Name, Rec)} + try Dict:'#get-'(Name, Rec) of + V -> + #diameter_avp{name = Name, value = value(Name, V)} catch error:_ -> undefined end. +value('Experimental-Result' = N, #{'Vendor-Id' := Vid, + 'Experimental-Result-Code' := RC}) -> + {N, Vid, RC}; +value('Experimental-Result' = N, [{'Experimental-Result-Code', RC}, + {'Vendor-Id', Vid}]) -> + {N, Vid, RC}; +value('Experimental-Result' = N, [{'Vendor-Id', Vid}, + {'Experimental-Result-Code', RC}]) -> + {N, Vid, RC}; +value(_, V) -> + V. + +%% find/2 + +find(Key, Map) + when is_map(Map) -> + maps:find(Key, Map); + +find(Key, List) + when is_list(List) -> + lists:keyfind(Key, 1, List). + %% get_avp_value/3 get_avp_value(Dict, Name, Msg) -> @@ -1909,18 +1989,25 @@ ungroup(Avp) -> %% avp_decode/3 +%% Ensure Experimental-Result is decoded as record, since this format +%% is used for counter keys. +avp_decode(Dict, 'Experimental-Result' = N, #diameter_avp{data = Bin} + = Avp) + when is_binary(Bin) -> + {V,_} = Dict:avp(decode, Bin, N, decode_opts(Dict)), + Avp#diameter_avp{name = N, value = V}; + avp_decode(Dict, Name, #diameter_avp{value = undefined, data = Bin} - = Avp) -> - try Dict:avp(decode, Bin, Name, decode_opts(Dict)) of - V -> - Avp#diameter_avp{value = V} - catch - error:_ -> - Avp - end; -avp_decode(_, _, #diameter_avp{} = Avp) -> - Avp. + = Avp) + when is_binary(Bin) -> + V = Dict:avp(decode, Bin, Name, decode_opts(Dict)), + Avp#diameter_avp{name = Name, value = V}; + +avp_decode(_, Name, #diameter_avp{} = Avp) -> + Avp#diameter_avp{name = Name}. + +%% cb/3 cb(#diameter_app{module = [_|_] = M}, F, A) -> eval(M, F, A). @@ -1933,7 +2020,9 @@ choose(false, _, X) -> X. %% Decode options sufficient for AVP extraction. decode_opts(Dict) -> - #{string_decode => false, + #{decode_format => record, + string_decode => false, strict_mbit => false, failed_avp => false, - dictionary => Dict}. + module => Dict, + app_dictionary => Dict}. diff --git a/lib/diameter/src/base/diameter_watchdog.erl b/lib/diameter/src/base/diameter_watchdog.erl index a63425d92a..c08e2da672 100644 --- a/lib/diameter/src/base/diameter_watchdog.erl +++ b/lib/diameter/src/base/diameter_watchdog.erl @@ -72,12 +72,12 @@ restrict := boolean(), suspect := non_neg_integer(), %% OKAY -> SUSPECT okay := non_neg_integer()}, %% REOPEN -> OKAY - codec :: #{string_decode := false, + codec :: #{decode_format := none, + string_decode := false, + strict_arities => diameter:strict_arities(), strict_mbit := boolean(), - failed_avp := false, rfc := 3588 | 6733, - ordered_encode := false, - incoming_maxlen := diameter:message_length()}, + ordered_encode := false}, shutdown = false :: boolean()}). %% --------------------------------------------------------------------------- @@ -135,13 +135,6 @@ i({Ack, T, Pid, {Opts, putr(restart, {T, Opts, Svc, SvcOpts}), %% save seeing it in trace putr(dwr, dwr(Caps)), %% Nodes = restrict_nodes(Restrict), - CodecKeys = [string_decode, - strict_mbit, - incoming_maxlen, - spawn_opt, - rfc, - ordered_encode], - #watchdog{parent = Pid, transport = start(T, Opts, SvcOpts, Nodes, Dict0, Svc), tw = proplists:get_value(watchdog_timer, @@ -149,14 +142,24 @@ i({Ack, T, Pid, {Opts, ?DEFAULT_TW_INIT), receive_data = RecvData, dictionary = Dict0, - config = - maps:without(CodecKeys, - config(SvcOpts#{restrict => restrict(Nodes), - suspect => 1, - okay => 3}, - Opts)), - codec = maps:with(CodecKeys, SvcOpts#{string_decode := false, - ordered_encode => false})}. + config = maps:with([sequence, + restrict_connections, + restrict, + suspect, + okay], + config(SvcOpts#{restrict => restrict(Nodes), + suspect => 1, + okay => 3}, + Opts)), + codec = maps:with([decode_format, + strict_arities, + strict_mbit, + string_decode, + rfc, + ordered_encode], + SvcOpts#{decode_format := none, + string_decode := false, + ordered_encode => false})}. wait(Ref, Pid) -> receive diff --git a/lib/diameter/src/compiler/diameter_codegen.erl b/lib/diameter/src/compiler/diameter_codegen.erl index f56e4a5249..4e6fe32d69 100644 --- a/lib/diameter/src/compiler/diameter_codegen.erl +++ b/lib/diameter/src/compiler/diameter_codegen.erl @@ -21,15 +21,14 @@ -module(diameter_codegen). %% -%% This module generates erl/hrl files for encode/decode modules -%% from the orddict parsed from a dictionary file (.dia) by -%% diameter_dict_util. The generated code is simple (one-liners), -%% the generated functions being called by code included iin the -%% generated modules from diameter_gen.hrl. The orddict itself is -%% returned by dict/0 in the generated module and diameter_dict_util -%% calls this function when importing dictionaries as a consequence -%% of @inherits sections. That is, @inherits introduces a dependency -%% on the beam file of another dictionary. +%% This module generates erl/hrl files for encode/decode modules from +%% the orddict parsed from a dictionary file by diameter_dict_util. +%% The generated code is simple (one-liners), and is called from +%% diameter_gen. The orddict itself is returned by dict/0 in the +%% generated module and diameter_dict_util calls this function when +%% importing dictionaries as a consequence of @inherits sections. That +%% is, @inherits introduces a dependency on the beam file of another +%% dictionary. %% -export([from_dict/4, diff --git a/lib/diameter/src/compiler/diameter_dict_util.erl b/lib/diameter/src/compiler/diameter_dict_util.erl index f9f2b02e94..7b53e51cb6 100644 --- a/lib/diameter/src/compiler/diameter_dict_util.erl +++ b/lib/diameter/src/compiler/diameter_dict_util.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -923,7 +923,7 @@ xa([D|_] = Ds, [[Qual, D, {_, Line, AvpName}] | Avps], Dict, Key, Name) -> store_new({Key, {Name, AvpName}}, [Line, Qual, D], Dict, - [Name, Line], + [AvpName, Line], avp_already_referenced), Key, Name); diff --git a/lib/diameter/src/compiler/diameter_exprecs.erl b/lib/diameter/src/compiler/diameter_exprecs.erl index 9a0cb6baf2..143dede037 100644 --- a/lib/diameter/src/compiler/diameter_exprecs.erl +++ b/lib/diameter/src/compiler/diameter_exprecs.erl @@ -110,9 +110,9 @@ %% parse_transform/2 parse_transform(Forms, _Options) -> - Rs = [R || {attribute, _, record, R} <- Forms], - Es = lists:append([E || {attribute, _, export_records, E} <- Forms]), {H,T} = lists:splitwith(fun is_head/1, Forms), + Rs = [R || {attribute, _, record, R} <- H], + Es = lists:append([E || {attribute, _, export_records, E} <- H]), H ++ [a_export(Es) | f_accessors(Es, Rs)] ++ T. is_head(T) -> diff --git a/lib/diameter/src/dict/doic_rfc7683.dia b/lib/diameter/src/dict/doic_rfc7683.dia new file mode 100644 index 0000000000..2b7804115e --- /dev/null +++ b/lib/diameter/src/dict/doic_rfc7683.dia @@ -0,0 +1,50 @@ +;; +;; %CopyrightBegin% +;; +;; Copyright Ericsson AB 2017. All Rights Reserved. +;; +;; Licensed under the Apache License, Version 2.0 (the "License"); +;; you may not use this file except in compliance with the License. +;; You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; +;; %CopyrightEnd% +;; + +@name diameter_gen_doic_rfc7683 +@prefix diameter_doic + +@avp_types + + OC-Supported-Features 621 Grouped - + OC-Feature-Vector 622 Unsigned64 - + OC-OLR 623 Grouped - + OC-Sequence-Number 624 Unsigned64 - + OC-Validity-Duration 625 Unsigned32 - + OC-Report-Type 626 Enumerated - + OC-Reduction-Percentage 627 Unsigned32 - + +@enum OC-Report-Type + + HOST_REPORT 0 + REALM_REPORT 1 + +@grouped + + OC-Supported-Features ::= < AVP Header: 621 > + [ OC-Feature-Vector ] + * [ AVP ] + + OC-OLR ::= < AVP Header: 623 > + < OC-Sequence-Number > + < OC-Report-Type > + [ OC-Reduction-Percentage ] + [ OC-Validity-Duration ] + * [ AVP ] diff --git a/lib/diameter/src/modules.mk b/lib/diameter/src/modules.mk index bb3b234d20..bb86de016a 100644 --- a/lib/diameter/src/modules.mk +++ b/lib/diameter/src/modules.mk @@ -24,6 +24,7 @@ DICTS = \ base_rfc6733 \ base_accounting \ acct_rfc6733 \ + doic_rfc7683 \ relay # The yecc grammar for the dictionary parser. diff --git a/lib/diameter/src/transport/diameter_sctp.erl b/lib/diameter/src/transport/diameter_sctp.erl index 6a9f1f940b..4eb3379d59 100644 --- a/lib/diameter/src/transport/diameter_sctp.erl +++ b/lib/diameter/src/transport/diameter_sctp.erl @@ -79,7 +79,7 @@ -type option() :: {sender, boolean()} | sender | {packet, boolean() | raw} - | {message_cb, false | diameter:evaluable()}. + | {message_cb, false | diameter:eval()}. -type uint() :: non_neg_integer(). @@ -102,9 +102,10 @@ streams :: {uint(), uint()} %% {InStream, OutStream} counts | undefined, os = 0 :: uint(), %% next output stream + rotate = 1 :: boolean() | 0 | 1, %% rotate os? packet = true :: boolean() %% legacy transport_data? | raw, - message_cb = false :: false | diameter:evaluable(), + message_cb = false :: false | diameter:eval(), send = false :: pid() | boolean()}). %% sending process %% Monitor process state. @@ -112,7 +113,7 @@ {transport :: pid(), ack = false :: boolean(), socket :: gen_sctp:sctp_socket(), - assoc_id :: gen_sctp:assoc_id()}). %% next output stream + assoc_id :: gen_sctp:assoc_id()}). %% Listener process state. -record(listener, @@ -120,7 +121,7 @@ socket :: gen_sctp:sctp_socket(), service :: pid(), %% service process pending = {0, queue:new()}, - opts :: [[match()] | boolean() | diameter:evaluable()]}). + opts :: [[match()] | boolean() | diameter:eval()]}). %% Field pending implements two queues: the first of transport-to-be %% processes to which an association has been assigned but for which %% diameter hasn't yet spawned a transport process, a short-lived @@ -156,12 +157,7 @@ start(T, Svc, Opts) = Svc, diameter_sctp_sup:start(), %% start supervisors on demand Addrs = Caps#diameter_caps.host_ip_address, - s(T, Addrs, Pid, lists:map(fun ip/1, Opts)). - -ip({ifaddr, A}) -> - {ip, A}; -ip(T) -> - T. + s(T, Addrs, Pid, Opts). %% A listener spawns transports either as a consequence of this call %% when there is not yet an association to assign it, or at comm_up on @@ -354,23 +350,35 @@ l([], Ref, T) -> %% open/3 open(Addrs, Opts, PortNr) -> - {LAs, Os} = addrs(Addrs, Opts), - {LAs, case gen_sctp:open(gen_opts(portnr(Os, PortNr))) of - {ok, Sock} -> - Sock; - {error, Reason} -> - x({open, Reason}) - end}. + case gen_sctp:open(gen_opts(portnr(addrs(Addrs, Opts), PortNr))) of + {ok, Sock} -> + {addrs(Sock), Sock}; + {error, Reason} -> + x({open, Reason}) + end. addrs(Addrs, Opts) -> - case proplists:split(Opts, [ip]) of - {[[]], _} -> - {Addrs, Opts ++ [{ip, A} || A <- Addrs]}; - {[As], Os} -> - LAs = [diameter_lib:ipaddr(A) || {ip, A} <- As], - {LAs, Os ++ [{ip, A} || A <- LAs]} + case lists:mapfoldl(fun ipaddr/2, false, Opts) of + {Os, true} -> + Os; + {_, false} -> + Opts ++ [{ip, A} || A <- Addrs] end. +ipaddr({K,A}, _) + when K == ifaddr; + K == ip -> + {{ip, ipaddr(A)}, true}; +ipaddr(T, B) -> + {T, B}. + +ipaddr(A) + when A == loopback; + A == any -> + A; +ipaddr(A) -> + diameter_lib:ipaddr(A). + portnr(Opts, PortNr) -> case proplists:get_value(port, Opts) of undefined -> @@ -379,6 +387,14 @@ portnr(Opts, PortNr) -> Opts end. +addrs(Sock) -> + case inet:socknames(Sock) of + {ok, As} -> + [A || {A,_} <- As]; + {error, Reason} -> + x({socknames, Reason}) + end. + %% x/1 x(Reason) -> @@ -565,7 +581,7 @@ transition(Msg, S) %% Deferred actions from a message_cb. transition({actions, Dir, Acts}, S) -> - actions(Acts, Dir, S); + setopts(ok, actions(Acts, Dir, S)); %% Request to close the transport connection. transition({diameter, {close, Pid}}, #transport{parent = Pid}) -> @@ -677,11 +693,16 @@ send(#diameter_packet{transport_data = {outstream, SId}} = S) -> send(SId rem OS, Msg, S); -%% ... or not: rotate through all streams. -send(Msg, #transport{streams = {_, OS}, +%% ... or not: rotate when sending on multiple streams ... +send(Msg, #transport{rotate = true, + streams = {_, OS}, os = N} = S) -> - send(N, Msg, S#transport{os = (N + 1) rem OS}). + send(N, Msg, S#transport{os = (N + 1) rem OS}); + +%% ... or send on the only stream available. +send(Msg, S) -> + send(0, Msg, S). %% send/3 @@ -749,7 +770,7 @@ recv({[#sctp_sndrcvinfo{assoc_id = Id}], _Bin} %% Inbound Diameter message. recv({[#sctp_sndrcvinfo{}], Bin} = Msg, S) when is_binary(Bin) -> - message(recv, Msg, S); + message(recv, Msg, recv(S)); recv({_, #sctp_shutdown_event{}}, _) -> stop; @@ -769,6 +790,25 @@ recv({_, #sctp_paddr_change{}}, _) -> recv({_, #sctp_pdapi_event{}}, _) -> ok. +%% recv/1 +%% +%% Start sending unordered after the second reception, so that an +%% outgoing CER/CEA will arrive at the peer before another request. + +recv(#transport{rotate = B} = S) + when is_boolean(B) -> + S; + +recv(#transport{rotate = 0, streams = {_,N}, socket = Sock} = S) -> + ok = inet:setopts(Sock, [{sctp_default_send_param, + #sctp_sndrcvinfo{flags = [unordered]}}]), + S#transport{rotate = 1 < N}; + +recv(#transport{rotate = N} = S) -> + S#transport{rotate = N-1}. + +%% publish/4 + publish(T, Ref, Id, Sock) -> true = diameter_reg:add_new({?MODULE, T, {Ref, {Id, Sock}}}), putr(?INFO_KEY, {gen_sctp, Sock}). %% for info/1 diff --git a/lib/diameter/src/transport/diameter_tcp.erl b/lib/diameter/src/transport/diameter_tcp.erl index a2f393d5d4..a8639baa11 100644 --- a/lib/diameter/src/transport/diameter_tcp.erl +++ b/lib/diameter/src/transport/diameter_tcp.erl @@ -87,8 +87,7 @@ module :: module() | undefined}). -type length() :: 0..16#FFFFFF. %% message length from Diameter header --type size() :: non_neg_integer(). %% accumulated binary size --type frag() :: {length(), size(), binary(), list(binary())} +-type frag() :: maybe_improper_list(length(), binary()) | binary(). -type connect_option() :: {raddr, inet:ip_address()} @@ -111,7 +110,7 @@ -type option() :: {port, non_neg_integer()} | {sender, boolean()} | sender - | {message_cb, false | diameter:evaluable()} + | {message_cb, false | diameter:eval()} | {fragment_timer, 0..16#FFFFFFFF}. %% Accepting/connecting transport process state. @@ -126,7 +125,7 @@ timeout :: infinity | 0..16#FFFFFFFF, %% fragment timeout tref = false :: false | reference(), %% fragment timer reference flush = false :: boolean(), %% flush fragment at timeout? - message_cb :: false | diameter:evaluable(), + message_cb :: false | diameter:eval(), send :: pid() | false}). %% sending process %% The usual transport using gen_tcp can be replaced by anything @@ -143,8 +142,7 @@ -> {ok, pid(), [inet:ip_address()]} when Ref :: diameter:transport_ref(); ({connect, Ref}, #diameter_service{}, [connect_option()]) - -> {ok, pid(), [inet:ip_address()]} - | {ok, pid()} + -> {ok, pid()} when Ref :: diameter:transport_ref(). start({T, Ref}, Svc, Opts) -> @@ -259,22 +257,14 @@ i(#monitor{parent = Pid, transport = TPid} = S) -> i({listen, Ref, {Mod, Opts, Addrs}}) -> [_] = diameter_config:subscribe(Ref, transport), %% assert existence - {[LA, LP], Rest} = proplists:split(Opts, [ip, port]), - LAddrOpt = get_addr(LA, Addrs), - LPort = get_port(LP), - {ok, LSock} = Mod:listen(LPort, gen_opts(LAddrOpt, Rest)), - LAddr = laddr(LAddrOpt, Mod, LSock), + {[LP], Rest} = proplists:split(Opts, [port]), + {ok, LSock} = Mod:listen(get_port(LP), gen_opts(Addrs, Rest)), + {ok, {LAddr, _}} = sockname(Mod, LSock), true = diameter_reg:add_new({?MODULE, listener, {Ref, {LAddr, LSock}}}), proc_lib:init_ack({ok, self(), {LAddr, LSock}}), #listener{socket = LSock, module = Mod}. -laddr([], Mod, Sock) -> - {ok, {Addr, _Port}} = sockname(Mod, Sock), - Addr; -laddr([{ip, Addr}], _, _) -> - Addr. - ssl_opts([]) -> false; ssl_opts([{ssl_options, true}]) -> @@ -309,24 +299,16 @@ init(accept = T, Ref, Mod, Pid, Opts, Addrs, SvcPid) -> Sock; init(connect = T, Ref, Mod, Pid, Opts, Addrs, _SvcPid) -> - {[LA, RA, RP], Rest} = proplists:split(Opts, [ip, raddr, rport]), - LAddrOpt = get_addr(LA, Addrs), + {[RA, RP], Rest} = proplists:split(Opts, [raddr, rport]), RAddr = get_addr(RA), RPort = get_port(RP), - proc_lib:init_ack(init_rc(LAddrOpt)), - Sock = ok(connect(Mod, RAddr, RPort, gen_opts(LAddrOpt, Rest))), + proc_lib:init_ack({ok, self()}), + Sock = ok(connect(Mod, RAddr, RPort, gen_opts(Addrs, Rest))), publish(Mod, T, Ref, Sock), - up(Pid, {RAddr, RPort}, LAddrOpt, Mod, Sock), + up(Pid, {RAddr, RPort}, Mod, Sock), Sock. -init_rc([{ip, Addr}]) -> - {ok, self(), [Addr]}; -init_rc([]) -> - {ok, self()}. - -up(Pid, Remote, [{ip, _Addr}], _, _) -> - diameter_peer:up(Pid, Remote); -up(Pid, Remote, [], Mod, Sock) -> +up(Pid, Remote, Mod, Sock) -> {Addr, _Port} = ok(sockname(Mod, Sock)), diameter_peer:up(Pid, Remote, [Addr]). @@ -383,25 +365,41 @@ l([{{?MODULE, listener, {_, AS}}, LPid}], _, _) -> l([], Ref, T) -> diameter_tcp_sup:start_child({listen, Ref, T}). -%% get_addr/1 +%% addrs/2 +%% +%% Take the first address from the service if several are specified +%% and not address is configured. + +addrs(Addrs, Opts) -> + case lists:mapfoldr(fun ipaddr/2, [], Opts) of + {Os, [_]} -> + Os; + {_, []} -> + Opts ++ [{ip, A} || [A|_] <- [Addrs]]; + {_, As} -> + ?ERROR({invalid_addrs, As, Addrs}) + end. -get_addr(As) -> - diameter_lib:ipaddr(addr(As, [])). +ipaddr({K,A}, As) + when K == ifaddr; + K == ip -> + {{ip, ipaddr(A)}, [A | As]}; +ipaddr(T, B) -> + {T, B}. -%% get_addr/2 +ipaddr(A) + when A == loopback; + A == any -> + A; +ipaddr(A) -> + diameter_lib:ipaddr(A). -get_addr([], []) -> - []; -get_addr(As, Def) -> - [{ip, diameter_lib:ipaddr(addr(As, Def))}]. +%% get_addr/1 -%% Take the first address from the service if several are unspecified. -addr([], [Addr | _]) -> - Addr; -addr([{_, Addr}], _) -> - Addr; -addr(As, Addrs) -> - ?ERROR({invalid_addrs, As, Addrs}). +get_addr([{_, Addr}]) -> + diameter_lib:ipaddr(Addr); +get_addr(Addrs) -> + ?ERROR({invalid_addrs, Addrs}). %% get_port/1 @@ -414,10 +412,15 @@ get_port(Ps) -> %% gen_opts/2 -gen_opts(LAddrOpt, Opts) -> +gen_opts(Addrs, Opts) -> + gen_opts(addrs(Addrs, Opts)). + +%% gen_opts/1 + +gen_opts(Opts) -> {L,_} = proplists:split(Opts, [binary, packet, active]), [[],[],[]] == L orelse ?ERROR({reserved_options, Opts}), - [binary, {packet, 0}, {active, false}] ++ LAddrOpt ++ Opts. + [binary, {packet, 0}, {active, false} | Opts]. %% --------------------------------------------------------------------------- %% # ports/1 @@ -599,11 +602,12 @@ t(T,S) -> %% Incoming packets. transition({P, Sock, Bin}, #transport{socket = Sock, - ssl = B} + ssl = B, + frag = Frag} = S) when P == ssl, true == B; P == tcp -> - recv(Bin, S#transport{active = false}); + recv(acc(Frag, Bin), S); %% Capabilties exchange has decided on whether or not to run over TLS. transition({diameter, {tls, Ref, Type, B}}, #transport{parent = Pid} @@ -640,7 +644,7 @@ transition(Msg, S) %% Deferred actions from a message_cb. transition({actions, Dir, Acts}, S) -> - actions(Acts, Dir, S); + setopts(actions(Acts, Dir, S)); %% Request to close the transport connection. transition({diameter, {close, Pid}}, #transport{parent = Pid, @@ -720,86 +724,77 @@ tls(accept, Sock, Opts) -> %% using Nagle. %% Receive packets until a full message is received, -recv(Bin, #transport{frag = Head} = S) -> - case rcv(Head, Bin) of - {Msg, B} -> %% have a complete message ... - message(recv, Msg, S#transport{frag = B}); - Frag -> %% read more on the socket - start_fragment_timer(setopts(S#transport{frag = Frag, - flush = false})) - end. -%% rcv/2 +recv({Msg, Rest}, S) -> %% have a complete message ... + recv(acc(Rest), message(recv, Msg, S)); + +recv(Frag, #transport{recv = B, + socket = Sock, + module = M} + = S) -> %% or not + B andalso setopts(M, Sock), + start_fragment_timer(S#transport{frag = Frag, + flush = false, + active = B}). -%% No previous fragment. -rcv(<<>>, Bin) -> - rcv(Bin); +%% acc/2 -%% Not even the first four bytes of the header. -rcv(Head, Bin) - when is_binary(Head) -> - rcv(<<Head/binary, Bin/binary>>); +%% Know how many bytes to extract. +acc([Len | Acc], Bin) -> + acc1(Len, <<Acc/binary, Bin/binary>>); -%% Or enough to know how many bytes to extract. -rcv({Len, N, Head, Acc}, Bin) -> - rcv(Len, N + size(Bin), Head, [Bin | Acc]). +%% Or not. +acc(Head, Bin) -> + acc(<<Head/binary, Bin/binary>>). -%% rcv/4 +%% acc1/3 %% Extract a message for which we have all bytes. -rcv(Len, N, Head, Acc) - when Len =< N -> - recv1(Len, bin(Head, Acc)); +acc1(Len, Bin) + when Len =< byte_size(Bin) -> + split_binary(Bin, Len); %% Wait for more packets. -rcv(Len, N, Head, Acc) -> - {Len, N, Head, Acc}. - -%% rcv/1 - -%% Nothing left. -rcv(<<>> = Bin) -> - Bin; - -%% The Message Length isn't even sufficient for a header. Chances are -%% things will go south from here but if we're lucky then the bytes we -%% have extend to an intended message boundary and we can recover by -%% simply receiving them. Make it so. -rcv(<<_:1/binary, Len:24, _/binary>> = Bin) - when Len < 20 -> - {Bin, <<>>}; - -%% Enough bytes to extract a message. -rcv(<<_:1/binary, Len:24, _/binary>> = Bin) - when Len =< size(Bin) -> - recv1(Len, Bin); - -%% Or not: wait for more packets. -rcv(<<_:1/binary, Len:24, _/binary>> = Head) -> - {Len, size(Head), Head, []}; +acc1(Len, Bin) -> + [Len | Bin]. + +%% acc/1 + +%% Don't match on Bin since this results in it being copied at the +%% next append according to the Efficiency Guide. This is also the +%% reason that the Len is extracted and maintained when accumulating +%% messages. The simplest implementation is just to accumulate a +%% binary and match <<_, Len:24, _/binary>> each time the length is +%% required, but the performance of this decays quadratically with the +%% message length, since the binary is then copied with each append of +%% additional bytes from gen_tcp. + +acc(Bin) + when 3 < byte_size(Bin) -> + {Head, _} = split_binary(Bin, 4), + [_,A,B,C] = binary_to_list(Head), + Len = (A bsl 16) bor (B bsl 8) bor C, + if Len < 20 -> + %% Message length isn't sufficient for a Diameter Header. + %% Chances are things will go south from here but if we're + %% lucky then the bytes we have extend to an intended + %% message boundary and we can recover by simply receiving + %% them. Make it so. + {Bin, <<>>}; + true -> + acc1(Len, Bin) + end; %% Not even 4 bytes yet. -rcv(Head) -> - Head. - -%% recv1/2 - -recv1(Len, Bin) -> - <<Msg:Len/binary, Rest/binary>> = Bin, - {Msg, Rest}. - -%% bin/2 - -bin(Head, Acc) -> - list_to_binary([Head | lists:reverse(Acc)]). +acc(Bin) -> + Bin. %% bin/1 -bin({_, _, Head, Acc}) -> - bin(Head, Acc); +bin([_ | Bin]) -> + Bin; -bin(Bin) - when is_binary(Bin) -> +bin(Bin) -> Bin. %% flush/1 @@ -911,14 +906,20 @@ setopts(#transport{socket = Sock, module = M} = S) when B, not A -> - case setopts(M, Sock, [{active, once}]) of - ok -> S#transport{active = true}; - X -> x({setopts, Sock, M, X}) %% possibly on peer disconnect - end; + setopts(M, Sock), + S#transport{active = true}; setopts(S) -> S. +%% setopts/2 + +setopts(M, Sock) -> + case setopts(M, Sock, [{active, once}]) of + ok -> ok; + X -> x({setopts, Sock, M, X}) %% possibly on peer disconnect + end. + %% portnr/2 portnr(gen_tcp, Sock) -> @@ -988,7 +989,7 @@ message(ack, _, #transport{message_cb = false} = S) -> S; message(Dir, Msg, #transport{message_cb = CB} = S) -> - recv(<<>>, actions(cb(CB, Dir, Msg), Dir, S)). + setopts(actions(cb(CB, Dir, Msg), Dir, S)). %% actions/3 diff --git a/lib/diameter/test/diameter_codec_SUITE.erl b/lib/diameter/test/diameter_codec_SUITE.erl index 9f08f49f9f..17112794e4 100644 --- a/lib/diameter/test/diameter_codec_SUITE.erl +++ b/lib/diameter/test/diameter_codec_SUITE.erl @@ -291,7 +291,8 @@ recode(Msg, Dict) -> recode(#diameter_packet{msg = Msg}, Dict). opts(Mod) -> - #{dictionary => Mod, + #{app_dictionary => Mod, + decode_format => record, string_decode => false, strict_mbit => true, rfc => 6733, diff --git a/lib/diameter/test/diameter_codec_SUITE_data/diameter_test_unknown.erl b/lib/diameter/test/diameter_codec_SUITE_data/diameter_test_unknown.erl index 700910878c..c6bba75f09 100644 --- a/lib/diameter/test/diameter_codec_SUITE_data/diameter_test_unknown.erl +++ b/lib/diameter/test/diameter_codec_SUITE_data/diameter_test_unknown.erl @@ -77,7 +77,8 @@ dec('BR', #diameter_packet ok. opts(Mod) -> - #{dictionary => Mod, + #{app_dictionary => Mod, + decode_format => record, string_decode => true, strict_mbit => true, rfc => 6733, diff --git a/lib/diameter/test/diameter_codec_test.erl b/lib/diameter/test/diameter_codec_test.erl index b548f85cb8..70e910ffa6 100644 --- a/lib/diameter/test/diameter_codec_test.erl +++ b/lib/diameter/test/diameter_codec_test.erl @@ -44,7 +44,8 @@ base() -> [] = run([[fun base/1, T] || T <- [zero, decode]]). gen(Mod) -> - Fs = [{Mod, F, []} || F <- [name, id, vendor_id, vendor_name]], + Fs = [{Mod, F, []} || Mod /= diameter_gen_doic_rfc7683, + F <- [name, id, vendor_id, vendor_name]], [] = run(Fs ++ [[fun gen/2, Mod, T] || T <- [messages, command_codes, avp_types, @@ -216,10 +217,11 @@ avp(Mod, encode = X, V, Name, _) -> opts(Mod) -> (opts())#{module => Mod, - dictionary => Mod}. + app_dictionary => Mod}. opts() -> - #{string_decode => true, + #{decode_format => record, + string_decode => true, strict_mbit => true, rfc => 6733, failed_avp => false}. diff --git a/lib/diameter/test/diameter_event_SUITE.erl b/lib/diameter/test/diameter_event_SUITE.erl index 57d3427037..a291dde6be 100644 --- a/lib/diameter/test/diameter_event_SUITE.erl +++ b/lib/diameter/test/diameter_event_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -63,7 +63,8 @@ {'Host-IP-Address', [?ADDR]}, {'Vendor-Id', 12345}, {'Product-Name', "OTP/diameter"}, - {'Acct-Application-Id', [D:id() || D <- Dicts]} + {'Acct-Application-Id', [D:id() || D <- Dicts]}, + {decode_format, map} | [{application, [{dictionary, D}, {module, #diameter_callback{}}]} || D <- Dicts]]). @@ -111,7 +112,8 @@ up(Config) -> {Svc, Ref} = connect(Config, [{connect_timer, 5000}, {watchdog_timer, 15000}]), start = event(Svc), - {up, Ref, {TPid, Caps}, Cfg, #diameter_packet{}} = event(Svc), + {up, Ref, {TPid, Caps}, Cfg, #diameter_packet{msg = M}} = event(Svc), + ['CEA' | #{}] = M, %% assert {watchdog, Ref, _, {initial, okay}, _} = event(Svc), %% Kill the transport process and see that the connection is %% reestablished after a watchdog timeout, not after connect_timer @@ -131,8 +133,9 @@ down(Config) -> {connect_timer, 5000}, {watchdog_timer, 20000}]), start = event(Svc), - {closed, Ref, {'CEA', ?NO_COMMON_APP, _, #diameter_packet{}}, _} + {closed, Ref, {'CEA', ?NO_COMMON_APP, _, #diameter_packet{msg = M}}, _} = event(Svc), + ['CEA' | #{}] = M, %% assert {reconnect, Ref, _} = event(Svc, 4000, 10000). %% Connect with matching capabilities but have the server delay its diff --git a/lib/diameter/test/diameter_examples_SUITE.erl b/lib/diameter/test/diameter_examples_SUITE.erl index eb99f10fe6..ee44ed8dc9 100644 --- a/lib/diameter/test/diameter_examples_SUITE.erl +++ b/lib/diameter/test/diameter_examples_SUITE.erl @@ -344,7 +344,7 @@ top(Dir, LibDir) -> start({server, Prot}) -> ok = diameter:start(), ok = server:start(), - {ok, Ref} = server:listen(Prot), + {ok, Ref} = server:listen({Prot, any, 3868}), [_] = ?util:lport(Prot, Ref), ok; @@ -352,7 +352,7 @@ start({client = Svc, Prot}) -> ok = diameter:start(), true = diameter:subscribe(Svc), ok = client:start(), - {ok, Ref} = client:connect(Prot), + {ok, Ref} = client:connect({Prot, loopback, loopback, 3868}), receive #diameter_event{info = {up, Ref, _, _, _}} -> ok end; start(Config) -> diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 84b41f14b7..ffb4a508cd 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -20,6 +20,7 @@ %% %% Tests of traffic between two Diameter nodes, one client, one server. +%% The traffic isn't meant to be sensible, just to exercise code. %% -module(diameter_traffic_SUITE). @@ -27,15 +28,18 @@ -export([suite/0, all/0, groups/0, + init_per_suite/0, init_per_suite/1, end_per_suite/1, + init_per_group/1, init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2]). %% testcases --export([start/1, +-export([rfc4005/1, + start/1, start_services/1, add_transports/1, result_codes/1, @@ -46,6 +50,7 @@ send_protocol_error/1, send_experimental_result/1, send_arbitrary/1, + send_proxy_info/1, send_unknown/1, send_unknown_short/1, send_unknown_mandatory/1, @@ -63,6 +68,7 @@ send_invalid_reject/1, send_unexpected_mandatory_decode/1, send_unexpected_mandatory/1, + send_too_many/1, send_long/1, send_maxlen/1, send_nopeer/1, @@ -98,18 +104,20 @@ stop/1]). %% diameter callbacks --export([peer_up/3, - peer_down/3, - pick_peer/6, pick_peer/7, - prepare_request/5, prepare_request/6, - prepare_retransmit/5, - handle_answer/6, handle_answer/7, - handle_error/6, - handle_request/3]). +-export([peer_up/4, + peer_down/4, + pick_peer/7, pick_peer/8, + prepare_request/6, prepare_request/7, + prepare_retransmit/6, + handle_answer/7, handle_answer/8, + handle_error/7, + handle_request/4]). %% diameter_{tcp,sctp} callbacks -export([message/3]). +-include_lib("kernel/include/inet_sctp.hrl"). + -include("diameter.hrl"). -include("diameter_gen_base_rfc3588.hrl"). -include("diameter_gen_base_accounting.hrl"). @@ -119,13 +127,22 @@ %% =========================================================================== +%% Fraction of shuffle/parallel groups to randomly skip. +-define(SKIP, 0.25). + +%% Positive number of testcases from which to select (randomly) from +%% tc(), the list of testcases to run, or [] to run all. The random +%% selection is to limit the time it takes for the suite to run. +-define(LIMIT, #{tcp => 42, sctp => 5}). + -define(util, diameter_util). -define(A, list_to_atom). -define(L, atom_to_list). +-define(B, iolist_to_binary). %% Don't use is_record/2 since dictionary hrl's aren't included. -%% (Since they define conflicting reqcords with the same names.) +%% (Since they define conflicting records with the same names.) -define(is_record(Rec, Name), (Name == element(1, Rec))). -define(ADDR, {127,0,0,1}). @@ -138,14 +155,14 @@ %% Sequence mask for End-to-End and Hop-by-Hop identifiers. -define(CLIENT_MASK, {1,26}). %% 1 in top 6 bits -%% How to construct messages, as record or list. --define(ENCODINGS, [list, record]). +%% How to construct outgoing messages. +-define(ENCODINGS, [list, record, map]). -%% How to send answers, in a diameter_packet or not. --define(CONTAINERS, [pkt, msg]). +%% How to decode incoming messages. +-define(DECODINGS, [record, none, map, list, record_from_map]). -%% Which common dictionary to use in the clients. --define(RFCS, [rfc3588, rfc6733]). +%% Which dictionary to use in the clients. +-define(RFCS, [rfc3588, rfc6733, rfc4005]). %% Whether to decode stringish Diameter types to strings, or leave %% them as binary. @@ -163,13 +180,12 @@ -record(group, {transport, strings, + encoding, client_service, - client_encoding, - client_dict0, + client_dict, client_sender, server_service, - server_encoding, - server_container, + server_decoding, server_sender, server_throttle}). @@ -182,34 +198,37 @@ %% A common match when receiving answers in a client. -define(answer_message(SessionId, ResultCode), - ['answer-message', - {'Session-Id', SessionId}, - {'Origin-Host', _}, - {'Origin-Realm', _}, - {'Result-Code', ResultCode} - | _]). + ['answer-message' | #{'Session-Id' := SessionId, + 'Origin-Host' := _, + 'Origin-Realm' := _, + 'Result-Code' := ResultCode}]). -define(answer_message(ResultCode), - ?answer_message(_, ResultCode)). + ['answer-message' | #{'Origin-Host' := _, + 'Origin-Realm' := _, + 'Result-Code' := ResultCode}]). %% Config for diameter:start_service/2. --define(SERVICE(Name, Decode), +-define(SERVICE(Name, Grp), [{'Origin-Host', Name ++ "." ++ ?REALM}, {'Origin-Realm', ?REALM}, {'Host-IP-Address', [?ADDR]}, {'Vendor-Id', 12345}, {'Product-Name', "OTP/diameter"}, - {'Auth-Application-Id', [?DIAMETER_APP_ID_COMMON]}, - {'Acct-Application-Id', [?DIAMETER_APP_ID_ACCOUNTING]}, + {'Auth-Application-Id', [0]}, %% common messages + {'Acct-Application-Id', [3]}, %% base accounting {restrict_connections, false}, - {string_decode, Decode}, + {string_decode, Grp#group.strings}, + {avp_dictionaries, [diameter_gen_doic_rfc7683]}, {incoming_maxlen, 1 bsl 21} | [{application, [{dictionary, D}, - {module, ?MODULE}, + {module, [?MODULE, Grp]}, {answer_errors, callback}]} || D <- [diameter_gen_base_rfc3588, diameter_gen_base_accounting, diameter_gen_base_rfc6733, - diameter_gen_acct_rfc6733]]]). + diameter_gen_acct_rfc6733, + nas4005], + D /= nas4005 orelse have_nas()]]). -define(SUCCESS, ?'DIAMETER_BASE_RESULT-CODE_SUCCESS'). @@ -227,6 +246,8 @@ ?'DIAMETER_BASE_RESULT-CODE_AVP_UNSUPPORTED'). -define(UNSUPPORTED_VERSION, ?'DIAMETER_BASE_RESULT-CODE_UNSUPPORTED_VERSION'). +-define(TOO_MANY, + ?'DIAMETER_BASE_RESULT-CODE_AVP_OCCURS_TOO_MANY_TIMES'). -define(REALM_NOT_SERVED, ?'DIAMETER_BASE_RESULT-CODE_REALM_NOT_SERVED'). -define(UNABLE_TO_DELIVER, @@ -254,64 +275,75 @@ suite() -> [{timetrap, {seconds, 10}}]. all() -> - [start, result_codes, {group, traffic}, empty, stop]. + [rfc4005, start, result_codes, {group, traffic}, empty, stop]. +%% Redefine this to run one or more groups for debugging purposes. +-define(GROUPS, []). +%-define(GROUPS, [[tcp,rfc6733,record,map,false,false,false,false]]). + +%% Issues with gen_sctp sporadically cause huge numbers of failed +%% testcases when running testcases in parallel. groups() -> - [{P, [P], Ts} || Ts <- [tc(tc())], P <- [shuffle, parallel]] + Names = names(), + [{P, [P], Ts} || Ts <- [tc()], P <- [shuffle, parallel]] ++ - [{?util:name([T,R,D,A,C,S,SS,ST,CS]), - [], - [{group, if S -> shuffle; not S -> parallel end}]} - || T <- ?TRANSPORTS, - R <- ?ENCODINGS, - D <- ?RFCS, - A <- ?ENCODINGS, - C <- ?CONTAINERS, - S <- ?STRING_DECODES, - SS <- ?SENDERS, - ST <- ?CALLBACKS, - CS <- ?SENDERS] + [{?util:name(N), [], [{group, if T == sctp; S -> shuffle; + true -> parallel end}]} + || [T,_,_,_,S|_] = N <- Names] ++ - [{T, [], groups([[T,R,D,A,C,S,SS,ST,CS] - || R <- ?ENCODINGS, - D <- ?RFCS, - A <- ?ENCODINGS, - C <- ?CONTAINERS, - S <- ?STRING_DECODES, - SS <- ?SENDERS, - ST <- ?CALLBACKS, - CS <- ?SENDERS, - SS orelse CS])} %% avoid deadlock + [{T, [], [{group, ?util:name(N)} || N <- names(Names, ?GROUPS), + T == hd(N)]} || T <- ?TRANSPORTS] ++ [{traffic, [], [{group, T} || T <- ?TRANSPORTS]}]. -%groups(_) -> %% debug -% Name = [sctp,record,rfc6733,record,pkt,false,false,false,false], -% [{group, ?util:name(Name)}]; -groups(Names) -> - [{group, ?util:name(L)} || L <- Names]. +names() -> + [[T,R,E,D,S,ST,SS,CS] || T <- ?TRANSPORTS, + R <- ?RFCS, + E <- ?ENCODINGS, + D <- ?DECODINGS, + S <- ?STRING_DECODES, + ST <- ?CALLBACKS, + SS <- ?SENDERS, + CS <- ?SENDERS]. + +names(Names, []) -> + [N || N <- Names, + [CS,SS|_] <- [lists:reverse(N)], + SS orelse CS]; %% avoid deadlock -%tc([N|_]) -> %% debug -% [N]; -tc(L) -> - L. +names(_, Names) -> + Names. %% -------------------- +init_per_suite() -> + [{timetrap, {seconds, 60}}]. + init_per_suite(Config) -> - [{sctp, ?util:have_sctp()} | Config]. + [{rfc4005, compile_and_load()}, {sctp, ?util:have_sctp()} | Config]. end_per_suite(_Config) -> + code:delete(nas4005), + code:purge(nas4005), ok. %% -------------------- +init_per_group(_) -> + [{timetrap, {seconds, 30}}]. + init_per_group(Name, Config) when Name == shuffle; Name == parallel -> - start_services(Config), - add_transports(Config); + case rand:uniform() < ?SKIP of + true -> + {skip, random}; + false -> + start_services(Config), + add_transports(Config), + replace({sleep, Name == parallel}, Config) + end; init_per_group(sctp = Name, Config) -> {_, Sctp} = lists:keyfind(Name, 1, Config), @@ -322,24 +354,22 @@ init_per_group(sctp = Name, Config) -> end; init_per_group(Name, Config) -> + Nas = proplists:get_value(rfc4005, Config, false), case ?util:name(Name) of - [T,R,D,A,C,S,SS,ST,CS] -> + [_,R,_,_,_,_,_,_] when R == rfc4005, true /= Nas -> + {skip, rfc4005}; + [T,R,E,D,S,ST,SS,CS] -> G = #group{transport = T, strings = S, + encoding = E, client_service = [$C|?util:unique_string()], - client_encoding = R, - client_dict0 = dict0(D), + client_dict = appdict(R), client_sender = CS, server_service = [$S|?util:unique_string()], - server_encoding = A, - server_container = C, + server_decoding = D, server_sender = SS, server_throttle = ST}, - %% Limit the number of testcase, since the number of - %% groups is large. - All = ?util:scramble(tc()), - TCs = lists:sublist(All, rand:uniform(32)), - [{group, G}, {runlist, TCs} | Config]; + replace([{group, G}, {runlist, select(T)}], Config); _ -> Config end. @@ -353,8 +383,26 @@ end_per_group(Name, Config) end_per_group(_, _) -> ok. +select(T) -> + try maps:get(T, ?LIMIT) of + N -> + lists:sublist(?util:scramble(tc()), max(5, rand:uniform(N))) + catch + error:_ -> ?LIMIT + end. + %% -------------------- +%% Work around common_test accumulating Config improperly, causing +%% testcases to get Config from groups and suites they're not in. +init_per_testcase(N, Config) + when N == rfc4005; + N == start; + N == result_codes; + N == empty; + N == stop -> + Config; + %% Skip testcases that can reasonably fail under SCTP. init_per_testcase(Name, Config) -> TCs = proplists:get_value(runlist, Config, []), @@ -368,12 +416,26 @@ init_per_testcase(Name, Config) -> _ when not Run -> {skip, random}; _ -> + proplists:get_value(sleep, Config, false) + andalso timer:sleep(rand:uniform(200)), [{testcase, Name} | Config] end. end_per_testcase(_, _) -> ok. +%% replace/2 +%% +%% Work around common_test running init functions inappropriately, and +%% this accumulating more config than expected. + +replace(Pairs, Config) + when is_list(Pairs) -> + lists:foldl(fun replace/2, Config, Pairs); + +replace({Key, _} = T, Config) -> + [T | lists:keydelete(Key, 1, Config)]. + %% -------------------- %% Testcases to run when services are started and connections @@ -386,6 +448,7 @@ tc() -> send_protocol_error, send_experimental_result, send_arbitrary, + send_proxy_info, send_unknown, send_unknown_short, send_unknown_mandatory, @@ -403,6 +466,7 @@ tc() -> send_invalid_reject, send_unexpected_mandatory_decode, send_unexpected_mandatory, + send_too_many, send_long, send_maxlen, send_nopeer, @@ -440,16 +504,26 @@ start(_Config) -> ok = diameter:start(). start_services(Config) -> - #group{strings = S, - client_service = CN, - server_service = SN} + #group{client_service = CN, + server_service = SN, + server_decoding = SD} + = Grp = group(Config), - ok = diameter:start_service(SN, ?SERVICE(SN, S)), - ok = diameter:start_service(CN, [{sequence, ?CLIENT_MASK} - | ?SERVICE(CN, S)]). + ok = diameter:start_service(SN, [{traffic_counters, bool()}, + {decode_format, SD} + | ?SERVICE(SN, Grp)]), + ok = diameter:start_service(CN, [{traffic_counters, bool()}, + {sequence, ?CLIENT_MASK}, + {decode_format, map}, + {strict_arities, decode} + | ?SERVICE(CN, Grp)]). + +bool() -> + 0.5 =< rand:uniform(). add_transports(Config) -> #group{transport = T, + encoding = E, client_service = CN, client_sender = CS, server_service = SN, @@ -459,30 +533,54 @@ add_transports(Config) -> LRef = ?util:listen(SN, [T, {sender, SS}, - {message_cb, ST andalso {?MODULE, message, [4]}} + {message_cb, ST andalso {?MODULE, message, [0]}} | [{packet, hd(?util:scramble([false, raw]))} || T == sctp andalso CS]], [{capabilities_cb, fun capx/2}, - {pool_size, 8}, - {applications, apps(rfc3588)}] + {pool_size, 8} + | server_apps()] ++ [{spawn_opt, {erlang, spawn, []}} || CS]), Cs = [?util:connect(CN, - [T, {sender, CS}], + [T, {sender, CS} | client_opts(T)], LRef, - [{id, Id}, - {capabilities, [{'Origin-State-Id', origin(Id)}]}, - {applications, apps(D)}]) - || A <- ?ENCODINGS, - C <- ?CONTAINERS, - D <- ?RFCS, - Id <- [{A,C}]], - %% The server uses the client's Origin-State-Id to decide how to - %% answer. + [{id, Id} + | client_apps(R, [{'Origin-State-Id', origin(Id)}])]) + || D <- ?DECODINGS, %% for multiple candidate peers + R <- ?RFCS, + R /= rfc4005 orelse have_nas(), + Id <- [{D,E}]], ?util:write_priv(Config, "transport", [LRef | Cs]). -apps(D0) -> - D = dict0(D0), - [acct(D), D]. +client_opts(tcp) -> + []; +client_opts(sctp) -> + [{sctp_initmsg, #sctp_initmsg{num_ostreams = N, + max_instreams = 5}} + || N <- [rand:uniform(8)], + N =< 6]. + +server_apps() -> + B = have_nas(), + [{applications, [diameter_gen_base_rfc3588, + diameter_gen_base_accounting] + ++ [nas4005 || B]}, + {capabilities, [{'Auth-Application-Id', [0] ++ [1 || B]}, %% common, NAS + {'Acct-Application-Id', [3]}]}]. %% accounting + +client_apps(D, Caps) -> + if D == rfc4005 -> + [{applications, [nas4005]}, + {capabilities, [{'Auth-Application-Id', [1]}, %% NAS + {'Acct-Application-Id', []} + | Caps]}]; + true -> + D0 = dict0(D), + [{applications, [acct(D0), D0]}, + {capabilities, Caps}] + end. + +have_nas() -> + false /= code:is_loaded(nas4005). remove_transports(Config) -> #group{client_service = CN, @@ -515,9 +613,16 @@ capx(_, #diameter_caps{origin_host = {OH,DH}}) -> %% =========================================================================== +%% Fail only this testcase if the RFC 4005 dictionary hasn't been +%% successfully compiled and loaded. +rfc4005(Config) -> + true = proplists:get_value(rfc4005, Config). + %% Ensure that result codes have the expected values. result_codes(_Config) -> - {2001, 3001, 3002, 3003, 3004, 3007, 3008, 3009, 5001, 5011, 5014} + {2001, + 3001, 3002, 3003, 3004, 3007, 3008, 3009, + 5001, 5009, 5011, 5014} = {?SUCCESS, ?COMMAND_UNSUPPORTED, ?UNABLE_TO_DELIVER, @@ -527,6 +632,7 @@ result_codes(_Config) -> ?INVALID_HDR_BITS, ?INVALID_AVP_BITS, ?AVP_UNSUPPORTED, + ?TOO_MANY, ?UNSUPPORTED_VERSION, ?INVALID_AVP_LENGTH}. @@ -534,8 +640,8 @@ result_codes(_Config) -> send_ok(Config) -> Req = ['ACR', {'Accounting-Record-Type', ?EVENT_RECORD}, {'Accounting-Record-Number', 1}], - - ['ACA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['ACA' | #{'Result-Code' := ?SUCCESS, + 'Session-Id' := _}] = call(Config, Req). %% Send an accounting ACR that the server answers badly to. @@ -551,7 +657,8 @@ send_eval(Config) -> Req = ['ACR', {'Accounting-Record-Type', ?EVENT_RECORD}, {'Accounting-Record-Number', 3}], - ['ACA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['ACA' | #{'Result-Code' := ?SUCCESS, + 'Session-Id' := _}] = call(Config, Req). %% Send an accounting ACR that the server tries to answer with an @@ -564,20 +671,87 @@ send_bad_answer(Config) -> = call(Config, Req). %% Send an ACR that the server callback answers explicitly with a -%% protocol error. +%% protocol error and some AVPs to check the decoding of. send_protocol_error(Config) -> Req = ['ACR', {'Accounting-Record-Type', ?EVENT_RECORD}, {'Accounting-Record-Number', 4}], - ?answer_message(?TOO_BUSY) - = call(Config, Req). + ['answer-message' | #{'Result-Code' := ?TOO_BUSY, + 'AVP' := [OLR | _]} = Avps] + = call(Config, Req), + + #diameter_avp{name = 'OC-OLR', + value = #{'OC-Sequence-Number' := 1, + 'OC-Report-Type' := 0, %% HOST_REPORT + 'OC-Reduction-Percentage' := [25], + 'OC-Validity-Duration' := [60], + 'AVP' := [OSF]}} + = OLR, + #diameter_avp{name = 'OC-Supported-Features', + value = #{} = Fs} + = OSF, + 0 = maps:size(Fs), + + #group{client_dict = D} = group(Config), + + if D == nas4005 -> + error = maps:find('Failed-AVP', Avps), + #{'AVP' := [_,Failed]} + = Avps, + #diameter_avp{name = 'Failed-AVP', + value = #{'AVP' := [NP,FR,AP]}} + = Failed, + #diameter_avp{name = 'NAS-Port', + value = 44} + = NP, + #diameter_avp{name = 'Firmware-Revision', + value = 12} + = FR, + #diameter_avp{name = 'Auth-Grace-Period', + value = 13} + = AP; + + D == diameter_gen_base_rfc3588; + D == diameter_gen_basr_accounting -> + error = maps:find('Failed-AVP', Avps), + #{'AVP' := [_,Failed]} + = Avps, + + #diameter_avp{name = 'Failed-AVP', + value = #{'AVP' := [NP,FR,AP]}} + = Failed, + #diameter_avp{name = undefined, + value = undefined} + = NP, + #diameter_avp{name = 'Firmware-Revision', + value = 12} + = FR, + #diameter_avp{name = 'Auth-Grace-Period', + value = 13} + = AP; + + D == diameter_gen_base_rfc6733; + D == diameter_gen_acct_rfc6733 -> + #{'Failed-AVP' := [#{'AVP' := [NP,FR,AP]}], + 'AVP' := [_]} + = Avps, + #diameter_avp{name = undefined, + value = undefined} + = NP, + #diameter_avp{name = 'Firmware-Revision', + value = 12} + = FR, + #diameter_avp{name = 'Auth-Grace-Period', + value = 13} + = AP + end. %% Send a 3xxx Experimental-Result in an answer not setting the E-bit %% and missing a Result-Code. send_experimental_result(Config) -> Req = ['ACR', {'Accounting-Record-Type', ?EVENT_RECORD}, {'Accounting-Record-Number', 5}], - ['ACA', {'Session-Id', _} | _] + ['ACA' | #{'Session-Id' := _}] = call(Config, Req). %% Send an ASR with an arbitrary non-mandatory AVP and expect success @@ -585,24 +759,37 @@ send_experimental_result(Config) -> send_arbitrary(Config) -> Req = ['ASR', {'AVP', [#diameter_avp{name = 'Product-Name', value = "XXX"}]}], - ['ASA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | Avps] + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS, + 'AVP' := [#diameter_avp{name = 'Product-Name', + value = V}]}] = call(Config, Req), - {'AVP', [#diameter_avp{name = 'Product-Name', - value = V}]} - = lists:last(Avps), "XXX" = string(V, Config). +%% Send Proxy-Info in an ASR that the peer answers with 3xxx, and +%% ensure that the AVP is returned. +send_proxy_info(Config) -> + H0 = ?B(?util:unique_string()), + S0 = ?B(?util:unique_string()), + Req = ['ASR', {'Proxy-Info', #{'Proxy-Host' => H0, + 'Proxy-State' => S0}}], + ['answer-message' | #{'Result-Code' := 3999, + 'Proxy-Info' := [#{'Proxy-Host' := H, + 'Proxy-State' := S}]}] + = call(Config, Req), + [H0, S0] = [?B(X) || X <- [H,S]]. + %% Send an unknown AVP (to some client) and check that it comes back. send_unknown(Config) -> Req = ['ASR', {'AVP', [#diameter_avp{code = 999, is_mandatory = false, data = <<17>>}]}], - ['ASA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | Avps] - = call(Config, Req), - {'AVP', [#diameter_avp{code = 999, - is_mandatory = false, - data = <<17>>}]} - = lists:last(Avps). + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS, + 'AVP' := [#diameter_avp{code = 999, + is_mandatory = false, + data = <<17>>}]}] + = call(Config, Req). %% Ditto, and point the AVP length past the end of the message. Expect %% 5014. @@ -613,28 +800,28 @@ send_unknown_short(Config, M, RC) -> Req = ['ASR', {'AVP', [#diameter_avp{code = 999, is_mandatory = M, data = <<17>>}]}], - ['ASA', {'Session-Id', _}, {'Result-Code', RC} | Avps] + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := RC, + 'Failed-AVP' := [#{'AVP' := [Avp]}]}] = call(Config, Req), - [#'diameter_base_Failed-AVP'{'AVP' = As}] - = proplists:get_value('Failed-AVP', Avps), - [#diameter_avp{code = 999, - is_mandatory = M, - data = <<17, _/binary>>}] %% extra bits from padding - = As. + #diameter_avp{code = 999, + is_mandatory = M, + data = <<17, _/binary>>} %% extra bits from padding + = Avp. %% Ditto but set the M flag. send_unknown_mandatory(Config) -> Req = ['ASR', {'AVP', [#diameter_avp{code = 999, is_mandatory = true, data = <<17>>}]}], - ['ASA', {'Session-Id', _}, {'Result-Code', ?AVP_UNSUPPORTED} | Avps] + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := ?AVP_UNSUPPORTED, + 'Failed-AVP' := [#{'AVP' := [Avp]}]}] = call(Config, Req), - [#'diameter_base_Failed-AVP'{'AVP' = As}] - = proplists:get_value('Failed-AVP', Avps), - [#diameter_avp{code = 999, - is_mandatory = true, - data = <<17>>}] - = As. + #diameter_avp{code = 999, + is_mandatory = true, + data = <<17>>} + = Avp. %% Ditto, and point the AVP length past the end of the message. Expect %% 5014 instead of 5001. @@ -647,15 +834,27 @@ send_unexpected_mandatory_decode(Config) -> Req = ['ASR', {'AVP', [#diameter_avp{code = 27, %% Session-Timeout is_mandatory = true, data = <<12:32>>}]}], - ['ASA', {'Session-Id', _}, {'Result-Code', ?AVP_UNSUPPORTED} | Avps] + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := ?AVP_UNSUPPORTED, + 'Failed-AVP' := [#{'AVP' := [Avp]}]}] = call(Config, Req), - [#'diameter_base_Failed-AVP'{'AVP' = As}] - = proplists:get_value('Failed-AVP', Avps), - [#diameter_avp{code = 27, - is_mandatory = true, - value = 12, - data = <<12:32>>}] - = As. + #diameter_avp{code = 27, + is_mandatory = true, + value = 12, + data = <<12:32>>} + = Avp. + +%% Try to two Auth-Application-Id in ASR expect 5009. +send_too_many(Config) -> + Req = ['ASR', {'Auth-Application-Id', [?APP_ID, 44]}], + + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := ?TOO_MANY, + 'Failed-AVP' := [#{'AVP' := [Avp]}]}] + = call(Config, Req), + #diameter_avp{name = 'Auth-Application-Id', + value = 44} + = Avp. %% Send an containing a faulty Grouped AVP (empty Proxy-Host in %% Proxy-Info) and expect that only the faulty AVP is sent in @@ -665,16 +864,13 @@ send_unexpected_mandatory_decode(Config) -> send_grouped_error(Config) -> Req = ['ASR', {'Proxy-Info', [[{'Proxy-Host', "abcd"}, {'Proxy-State', ""}]]}], - ['ASA', {'Session-Id', _}, {'Result-Code', ?INVALID_AVP_LENGTH} | Avps] + ['ASA' | #{'Session-Id' := _, + 'Result-Code' := ?INVALID_AVP_LENGTH, + 'Failed-AVP' := [#{'AVP' := [Avp]}]}] = call(Config, Req), - [#'diameter_base_Failed-AVP'{'AVP' = As}] - = proplists:get_value('Failed-AVP', Avps), - [#diameter_avp{name = 'Proxy-Info', - value = #'diameter_base_Proxy-Info' - {'Proxy-Host' = Empty, - 'Proxy-State' = undefined}}] - = As, - <<0>> = iolist_to_binary(Empty). + #diameter_avp{name = 'Proxy-Info', value = #{'Proxy-Host' := H}} + = Avp, + <<0>> = ?B(H). %% Send an STR that the server ignores. send_noreply(Config) -> @@ -702,7 +898,8 @@ send_error_bit(Config) -> %% Send a bad version and check that we get 5011. send_unsupported_version(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}], - ['STA', {'Session-Id', _}, {'Result-Code', ?UNSUPPORTED_VERSION} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?UNSUPPORTED_VERSION}] = call(Config, Req). %% Send a request containing an AVP length > data size. @@ -722,16 +919,11 @@ send_zero_avp_length(Config) -> send_invalid_avp_length(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}], - ['STA', {'Session-Id', _}, - {'Result-Code', ?INVALID_AVP_LENGTH}, - {'Origin-Host', _}, - {'Origin-Realm', _}, - {'User-Name', _}, - {'Class', _}, - {'Error-Message', _}, - {'Error-Reporting-Host', _}, - {'Failed-AVP', [#'diameter_base_Failed-AVP'{'AVP' = [_]}]} - | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?INVALID_AVP_LENGTH, + 'Origin-Host' := _, + 'Origin-Realm' := _, + 'Failed-AVP' := [#{'AVP' := [_]}]}] = call(Config, Req). %% Send a request containing 5xxx errors that the server rejects with @@ -747,14 +939,16 @@ send_invalid_reject(Config) -> send_unexpected_mandatory(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}], - ['STA', {'Session-Id', _}, {'Result-Code', ?AVP_UNSUPPORTED} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?AVP_UNSUPPORTED}] = call(Config, Req). %% Send something long that will be fragmented by TCP. send_long(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}, {'User-Name', [binary:copy(<<$X>>, 1 bsl 20)]}], - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = call(Config, Req). %% Send something longer than the configure incoming_maxlen. @@ -797,7 +991,8 @@ send_any_2(Config) -> send_all_1(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}], Realm = lists:foldr(fun(C,A) -> [C,A] end, [], ?REALM), - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = call(Config, Req, [{filter, {all, [{host, any}, {realm, Realm}]}}]). send_all_2(Config) -> @@ -826,13 +1021,13 @@ send_detach(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}], Ref = make_ref(), ok = call(Config, Req, [{extra, [{self(), Ref}]}, detach]), - Ans = receive {Ref, T} -> T end, - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] - = Ans. + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] + = receive {Ref, T} -> T end. %% Send a request which can't be encoded and expect {error, encode}. send_encode_error(Config) -> - {error, encode} = call(Config, ['STR']). %% No Termination-Cause + {error, encode} = call(Config, ['STR', {'Termination-Cause', huh}]). %% Send with filtering and expect success. send_destination_1(Config) -> @@ -840,25 +1035,27 @@ send_destination_1(Config) -> = group(Config), Req = ['STR', {'Termination-Cause', ?LOGOUT}, {'Destination-Host', [?HOST(SN, ?REALM)]}], - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = call(Config, Req, [{filter, {all, [host, realm]}}]). send_destination_2(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}], - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = call(Config, Req, [{filter, {all, [host, realm]}}]). %% Send with filtering on and expect failure when specifying an %% unknown host or realm. send_destination_3(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}, - {'Destination-Realm', "unknown.org"}], + {'Destination-Realm', <<"unknown.org">>}], {error, no_connection} = call(Config, Req, [{filter, {all, [host, realm]}}]). send_destination_4(Config) -> #group{server_service = SN} = group(Config), Req = ['STR', {'Termination-Cause', ?LOGOUT}, - {'Destination-Host', [?HOST(SN, "unknown.org")]}], + {'Destination-Host', [?HOST(SN, ["unknown.org"])]}], {error, no_connection} = call(Config, Req, [{filter, {all, [host, realm]}}]). @@ -866,7 +1063,7 @@ send_destination_4(Config) -> %% an unknown host or realm. send_destination_5(Config) -> Req = ['STR', {'Termination-Cause', ?LOGOUT}, - {'Destination-Realm', "unknown.org"}], + {'Destination-Realm', [<<"unknown.org">>]}], ?answer_message(?REALM_NOT_SERVED) = call(Config, Req). send_destination_6(Config) -> @@ -908,7 +1105,8 @@ send_bad_filter(Config, F) -> %% Specify multiple filter options and expect them be conjunctive. send_multiple_filters_1(Config) -> Fun = fun(#diameter_caps{}) -> true end, - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = send_multiple_filters(Config, [host, {eval, Fun}]). send_multiple_filters_2(Config) -> E = {erlang, is_tuple, []}, @@ -919,7 +1117,8 @@ send_multiple_filters_3(Config) -> E2 = {erlang, is_tuple, []}, E3 = {erlang, is_record, [diameter_caps]}, E4 = [{erlang, is_record, []}, diameter_caps], - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = send_multiple_filters(Config, [{eval, E} || E <- [E1,E2,E3,E4]]). send_multiple_filters(Config, Fs) -> @@ -930,7 +1129,8 @@ send_multiple_filters(Config, Fs) -> %% only the return value from the prepare_request callback being %% significant. send_anything(Config) -> - ['STA', {'Session-Id', _}, {'Result-Code', ?SUCCESS} | _] + ['STA' | #{'Session-Id' := _, + 'Result-Code' := ?SUCCESS}] = call(Config, anything). %% =========================================================================== @@ -954,58 +1154,137 @@ call(Config, Req) -> call(Config, Req, Opts) -> Name = proplists:get_value(testcase, Config), - #group{client_service = CN, - client_encoding = ReqEncoding, - client_dict0 = Dict0} - = Group + #group{encoding = Enc, + client_service = CN, + client_dict = Dict0} = group(Config), diameter:call(CN, dict(Req, Dict0), - msg(Req, ReqEncoding, Dict0), - [{extra, [{Name, Group}, diameter_lib:now()]} | Opts]). + msg(Req, Enc, Dict0), + [{extra, [Name, diameter_lib:now()]} | Opts]). -origin({A,C}) -> - 2*codec(A) + container(C); +origin({D,E}) -> + 4*decode(D) + encode(E); origin(N) -> - {codec(N band 2), container(N rem 2)}. - -%% Map booleans, but the readable atoms are part of (constructed) -%% group names, so it's good that they're readable. - -codec(record) -> 0; -codec(list) -> 1; -codec(0) -> record; -codec(_) -> list. - -container(pkt) -> 0; -container(msg) -> 1; -container(0) -> pkt; -container(_) -> msg. + {decode(N bsr 2), encode(N rem 4)}. + +%% Map atoms. The atoms are part of (constructed) group names, so it's +%% good that they're readable. + +decode(record) -> 0; +decode(list) -> 1; +decode(map) -> 2; +decode(none) -> 3; +decode(record_from_map) -> 4; +decode(0) -> record; +decode(1) -> list; +decode(2) -> map; +decode(3) -> none; +decode(4) -> record_from_map. + +encode(record) -> 0; +encode(list) -> 1; +encode(map) -> 2; +encode(0) -> record; +encode(1) -> list; +encode(2) -> map. msg([H|_] = Msg, record = E, diameter_gen_base_rfc3588) when H == 'ACR'; H == 'ACA' -> msg(Msg, E, diameter_gen_base_accounting); + msg([H|_] = Msg, record = E, diameter_gen_base_rfc6733) when H == 'ACR'; H == 'ACA' -> msg(Msg, E, diameter_gen_acct_rfc6733); + msg([H|T], record, Dict) -> Dict:'#new-'(Dict:msg2rec(H), T); + +msg([H|As], map, _) + when is_list(As) -> + [H | maps:from_list(As)]; + msg(Msg, _, _) -> Msg. +to_map(#diameter_packet{msg = [_MsgName | Avps] = Msg}, + #group{server_decoding = map}) + when is_map(Avps) -> + Msg; + +to_map(#diameter_packet{msg = [MsgName | Avps]}, + #group{server_decoding = list}) -> + [MsgName | maps:from_list(Avps)]; + +to_map(#diameter_packet{header = H, msg = Rec}, + #group{server_decoding = D}) + when D == record; + D == record_from_map -> + rec_to_map(Rec, dict(H)); + +%% No record decode: do it ourselves. +to_map(#diameter_packet{header = H, + msg = Name, + bin = Bin}, + #group{server_decoding = none, + strings = B}) -> + Opts = #{decode_format => map, + string_decode => B, + avp_dictionaries => [diameter_gen_doic_rfc7683], + strict_mbit => true, + rfc => 6733}, + #diameter_packet{msg = [MsgName | _Map] = Msg} + = diameter_codec:decode(dict(H), Opts, Bin), + {MsgName, _} = {Name, Msg}, %% assert + Msg. + +dict(#diameter_header{application_id = Id, + cmd_code = Code}) -> + if Id == 1 -> + nas4005; + Code == 271 -> + diameter_gen_base_accounting; + true -> + diameter_gen_base_rfc3588 + end. + +rec_to_map(Rec, Dict) -> + [R | Vs] = Dict:'#get-'(Rec), + [Dict:rec2msg(R) | maps:from_list([T || {_,V} = T <- Vs, + V /= undefined, + V /= []])]. + +appdict(rfc4005) -> + nas4005; +appdict(D) -> + dict0(D). + dict0(D) -> ?A("diameter_gen_base_" ++ ?L(D)). -dict(Msg, Dict0) - when 'ACR' == hd(Msg); - 'ACA' == hd(Msg); - ?is_record(Msg, diameter_base_accounting_ACR); - ?is_record(Msg, diameter_base_accounting_ACA) -> +dict(Msg, Dict) -> + d(name(Msg), Dict). + +d(N, nas4005 = D) -> + if N == {list, 'answer-message'}; + N == {map, 'answer-message'}; + N == {record, 'diameter_base_answer-message'} -> + diameter_gen_base_rfc3588; + true -> + D + end; +d(N, Dict0) + when N == {list, 'ACR'}; + N == {list, 'ACA'}; + N == {map, 'ACR'}; + N == {map, 'ACA'}; + N == {record, diameter_base_accounting_ACR}; + N == {record, diameter_base_accounting_ACA} -> acct(Dict0); -dict(_, Dict0) -> +d(_, Dict0) -> Dict0. acct(diameter_gen_base_rfc3588) -> @@ -1014,53 +1293,60 @@ acct(diameter_gen_base_rfc6733) -> diameter_gen_acct_rfc6733. %% Set only values that aren't already. -set(_, [H|T], Vs) -> - [H | Vs ++ T]; -set(#group{client_dict0 = Dict0} = _Group, Rec, Vs) -> + +set(_, [N | As], Vs) -> + [N | if is_map(As) -> + maps:merge(maps:from_list(Vs), As); + is_list(As) -> + Vs ++ As + end]; + +set(#group{client_dict = Dict0} = _Group, Rec, Vs) -> Dict = dict(Rec, Dict0), lists:foldl(fun({F,_} = FV, A) -> - set(Dict, Dict:'#get-'(F, A), FV, A) + reset(Dict, Dict:'#get-'(F, A), FV, A) end, Rec, Vs). -set(Dict, E, FV, Rec) +reset(Dict, E, FV, Rec) when E == undefined; E == [] -> Dict:'#set-'(FV, Rec); -set(_, _, _, Rec) -> + +reset(_, _, _, Rec) -> Rec. %% =========================================================================== %% diameter callbacks -%% peer_up/3 +%% peer_up/4 -peer_up(_SvcName, _Peer, State) -> +peer_up(_SvcName, _Peer, State, _Group) -> State. %% peer_down/3 -peer_down(_SvcName, _Peer, State) -> +peer_down(_SvcName, _Peer, State, _Group) -> State. -%% pick_peer/6-7 +%% pick_peer/7-8 -pick_peer(Peers, _, [$C|_], _State, {Name, Group}, _) +pick_peer(Peers, _, [$C|_], _State, Group, Name, _) when Name /= send_detach -> find(Group, Peers). -pick_peer(_Peers, _, [$C|_], _State, {send_nopeer, _}, _, ?EXTRA) -> +pick_peer(_Peers, _, [$C|_], _State, _Group, send_nopeer, _, ?EXTRA) -> false; -pick_peer(Peers, _, [$C|_], _State, {send_detach, Group}, _, {_,_}) -> +pick_peer(Peers, _, [$C|_], _State, Group, send_detach, _, {_,_}) -> find(Group, Peers). -find(#group{client_service = CN, - server_encoding = A, - server_container = C}, +find(#group{encoding = E, + client_service = CN, + server_decoding = D}, [_|_] = Peers) -> - Id = {A,C}, + Id = {D,E}, [P] = [P || P <- Peers, id(Id, P, CN)], {ok, P}. @@ -1069,15 +1355,15 @@ id(Id, {Pid, _Caps}, SvcName) -> = diameter:service_info(SvcName, Pid), lists:member({id, Id}, Opts). -%% prepare_request/5-6 +%% prepare_request/6-7 -prepare_request(_Pkt, [$C|_], {_Ref, _Caps}, {send_discard, _}, _) -> +prepare_request(_Pkt, [$C|_], {_Ref, _Caps}, _, send_discard, _) -> {discard, unprepared}; -prepare_request(Pkt, [$C|_], {_Ref, Caps}, {Name, Group}, _) -> +prepare_request(Pkt, [$C|_], {_Ref, Caps}, Group, Name, _) -> {send, prepare(Pkt, Caps, Name, Group)}. -prepare_request(Pkt, [$C|_], {_Ref, Caps}, {send_detach, Group}, _, _) -> +prepare_request(Pkt, [$C|_], {_Ref, Caps}, Group, send_detach, _, _) -> {eval_packet, {send, prepare(Pkt, Caps, Group)}, [fun log/2, detach]}. log(#diameter_packet{bin = Bin} = P, T) @@ -1086,7 +1372,7 @@ log(#diameter_packet{bin = Bin} = P, T) %% prepare/4 -prepare(Pkt, Caps, N, #group{client_dict0 = Dict0} = Group) +prepare(Pkt, Caps, N, #group{client_dict = Dict0} = Group) when N == send_unknown_short_mandatory; N == send_unknown_short -> Req = prepare(Pkt, Caps, Group), @@ -1106,7 +1392,7 @@ prepare(Pkt, Caps, N, #group{client_dict0 = Dict0} = Group) <<H:Offset/binary, Len:24, T/binary>> = Bin, E#diameter_packet{bin = <<H/binary, (Len+9):24, T/binary>>}; -prepare(Pkt, Caps, N, #group{client_dict0 = Dict0} = Group) +prepare(Pkt, Caps, N, #group{client_dict = Dict0} = Group) when N == send_long_avp_length; N == send_short_avp_length; N == send_zero_avp_length -> @@ -1132,7 +1418,7 @@ prepare(Pkt, Caps, N, #group{client_dict0 = Dict0} = Group) T/binary, Hdr/binary, AL:24, Data/binary>>}; -prepare(Pkt, Caps, N, #group{client_dict0 = Dict0} = Group) +prepare(Pkt, Caps, N, #group{client_dict = Dict0} = Group) when N == send_invalid_avp_length; N == send_invalid_reject -> Req = prepare(Pkt, Caps, Group), @@ -1147,7 +1433,7 @@ prepare(Pkt, Caps, N, #group{client_dict0 = Dict0} = Group) <<V, L:24, H/binary>> = H0, %% assert E#diameter_packet{bin = <<V, (L+4):24, H/binary, 16:24, 0:32, T/binary>>}; -prepare(Pkt, Caps, send_unexpected_mandatory, #group{client_dict0 = Dict0} +prepare(Pkt, Caps, send_unexpected_mandatory, #group{client_dict = Dict0} = Group) -> Req = prepare(Pkt, Caps, Group), #diameter_packet{bin = <<V, Len:24, T/binary>>} @@ -1157,7 +1443,7 @@ prepare(Pkt, Caps, send_unexpected_mandatory, #group{client_dict0 = Dict0} Avp = <<Code:32, Flags, 8:24>>, E#diameter_packet{bin = <<V, (Len+8):24, T/binary, Avp/binary>>}; -prepare(Pkt, Caps, send_grouped_error, #group{client_dict0 = Dict0} +prepare(Pkt, Caps, send_grouped_error, #group{client_dict = Dict0} = Group) -> Req = prepare(Pkt, Caps, Group), #diameter_packet{bin = Bin} @@ -1189,14 +1475,14 @@ prepare(Pkt, Caps, send_grouped_error, #group{client_dict0 = Dict0} Payload/binary, T/binary>>}; -prepare(Pkt, Caps, send_unsupported, #group{client_dict0 = Dict0} = Group) -> +prepare(Pkt, Caps, send_unsupported, #group{client_dict = Dict0} = Group) -> Req = prepare(Pkt, Caps, Group), #diameter_packet{bin = <<H:5/binary, _CmdCode:3/binary, T/binary>>} = E = diameter_codec:encode(Dict0, Pkt#diameter_packet{msg = Req}), E#diameter_packet{bin = <<H/binary, 42:24, T/binary>>}; -prepare(Pkt, Caps, send_unsupported_app, #group{client_dict0 = Dict0} +prepare(Pkt, Caps, send_unsupported_app, #group{client_dict = Dict0} = Group) -> Req = prepare(Pkt, Caps, Group), #diameter_packet{bin = <<H:8/binary, _ApplId:4/binary, T/binary>>} @@ -1223,93 +1509,120 @@ prepare(Pkt, Caps, _Name, Group) -> %% prepare/3 -prepare(#diameter_packet{msg = Req}, Caps, Group) - when ?is_record(Req, diameter_base_accounting_ACR); - 'ACR' == hd(Req) -> +prepare(#diameter_packet{msg = Req} = Pkt, Caps, Group) -> + set(name(Req), Pkt, Caps, Group). + +%% set/4 + +set(N, #diameter_packet{msg = Req}, Caps, Group) + when N == {record, diameter_base_accounting_ACR}; + N == {record, nas_ACR}; + N == {map, 'ACR'}; + N == {list, 'ACR'} -> #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, DR}} = Caps, - set(Group, Req, [{'Session-Id', diameter:session_id(OH)}, - {'Origin-Host', OH}, - {'Origin-Realm', OR}, - {'Destination-Realm', DR}]); + set(Group, Req, [{'Session-Id', [diameter:session_id(OH)]}, + {'Origin-Host', [OH]}, + {'Origin-Realm', [OR]}, + {'Destination-Realm', [DR]}]); -prepare(#diameter_packet{msg = Req}, Caps, Group) - when ?is_record(Req, diameter_base_ASR); - 'ASR' == hd(Req) -> +set(N, #diameter_packet{msg = Req}, Caps, Group) + when N == {record, diameter_base_ASR}; + N == {record, nas_ASR}; + N == {map, 'ASR'}; + N == {list, 'ASR'} -> #diameter_caps{origin_host = {OH, DH}, origin_realm = {OR, DR}} = Caps, - set(Group, Req, [{'Session-Id', diameter:session_id(OH)}, - {'Origin-Host', OH}, - {'Origin-Realm', OR}, - {'Destination-Host', DH}, - {'Destination-Realm', DR}, + set(Group, Req, [{'Session-Id', [diameter:session_id(OH)]}, + {'Origin-Host', [OH]}, + {'Origin-Realm', [OR]}, + {'Destination-Host', [DH]}, + {'Destination-Realm', [DR]}, {'Auth-Application-Id', ?APP_ID}]); -prepare(#diameter_packet{msg = Req}, Caps, Group) - when ?is_record(Req, diameter_base_STR); - 'STR' == hd(Req) -> +set(N, #diameter_packet{msg = Req}, Caps, Group) + when N == {record, diameter_base_STR}; + N == {record, nas_STR}; + N == {map, 'STR'}; + N == {list, 'STR'} -> #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, DR}} = Caps, - set(Group, Req, [{'Session-Id', diameter:session_id(OH)}, - {'Origin-Host', OH}, - {'Origin-Realm', OR}, - {'Destination-Realm', DR}, + set(Group, Req, [{'Session-Id', [diameter:session_id(OH)]}, + {'Origin-Host', [OH]}, + {'Origin-Realm', [OR]}, + {'Destination-Realm', [DR]}, {'Auth-Application-Id', ?APP_ID}]); -prepare(#diameter_packet{msg = Req}, Caps, Group) - when ?is_record(Req, diameter_base_RAR); - 'RAR' == hd(Req) -> +set(N, #diameter_packet{msg = Req}, Caps, Group) + when N == {record, diameter_base_RAR}; + N == {record, nas_RAR}; + N == {map, 'RAR'}; + N == {list, 'RAR'} -> #diameter_caps{origin_host = {OH, DH}, origin_realm = {OR, DR}} = Caps, - set(Group, Req, [{'Session-Id', diameter:session_id(OH)}, - {'Origin-Host', OH}, - {'Origin-Realm', OR}, - {'Destination-Host', DH}, - {'Destination-Realm', DR}, + set(Group, Req, [{'Session-Id', [diameter:session_id(OH)]}, + {'Origin-Host', [OH]}, + {'Origin-Realm', [OR]}, + {'Destination-Host', [DH]}, + {'Destination-Realm', [DR]}, {'Auth-Application-Id', ?APP_ID}]). -%% prepare_retransmit/5 +%% name/1 + +name([H|#{}]) -> + {map, H}; + +name([H|_]) -> + {list, H}; -prepare_retransmit(_Pkt, false, _Peer, _Name, _Group) -> +name(Rec) -> + try + {record, element(1, Rec)} + catch + error: badarg -> + false + end. + +%% prepare_retransmit/6 + +prepare_retransmit(_Pkt, false, _Peer, _Group, _Name, _) -> discard. -%% handle_answer/6-7 +%% handle_answer/7-8 -handle_answer(Pkt, Req, [$C|_], Peer, {Name, Group}, _) -> +handle_answer(Pkt, Req, [$C|_], Peer, Group, Name, _) -> answer(Pkt, Req, Peer, Name, Group). -handle_answer(Pkt, Req, [$C|_], Peer, {send_detach = Name, Group}, _, X) -> +handle_answer(Pkt, Req, [$C|_], Peer, Group, send_detach = Name, _, X) -> {Pid, Ref} = X, Pid ! {Ref, answer(Pkt, Req, Peer, Name, Group)}. -answer(Pkt, Req, _Peer, Name, #group{client_dict0 = Dict0}) -> +answer(Pkt, Req, _Peer, Name, #group{client_dict = Dict0}) -> #diameter_packet{header = H, msg = Ans, errors = Es} = Pkt, ApplId = app(Req, Name, Dict0), #diameter_header{application_id = ApplId} = H, %% assert - Dict = dict(Ans, Dict0), - [R | Vs] = Dict:'#get-'(answer(Ans, Es, Name)), - [Dict:rec2msg(R) | Vs]. + answer(Ans, Es, Name). %% Missing Result-Code and inappropriate Experimental-Result-Code. -answer(Rec, Es, send_experimental_result) -> +answer(Ans, Es, send_experimental_result) -> [{5004, #diameter_avp{name = 'Experimental-Result'}}, {5005, #diameter_avp{name = 'Result-Code'}}] = Es, - Rec; + Ans; %% An inappropriate E-bit results in a decode error ... -answer(Rec, Es, send_bad_answer) -> +answer(Ans, Es, send_bad_answer) -> [{5004, #diameter_avp{name = 'Result-Code'}} | _] = Es, - Rec; + Ans; %% ... while other errors are reflected in Failed-AVP. -answer(Rec, [], _) -> - Rec. +answer(Ans, [], _) -> + Ans. app(_, send_unsupported_app, _) -> ?BAD_APP; @@ -1317,25 +1630,29 @@ app(Req, _, Dict0) -> Dict = dict(Req, Dict0), Dict:id(). -%% handle_error/6 +%% handle_error/7 -handle_error(timeout = Reason, _Req, [$C|_], _Peer, _, Time) -> +handle_error(timeout = Reason, _Req, [$C|_], _Peer, _, _, Time) -> Now = diameter_lib:now(), {Reason, {diameter_lib:timestamp(Time), diameter_lib:timestamp(Now), diameter_lib:micro_diff(Now, Time)}}; -handle_error(Reason, _Req, [$C|_], _Peer, _, _Time) -> +handle_error(Reason, _Req, [$C|_], _Peer, _, _, _Time) -> {error, Reason}. -%% handle_request/3 +%% handle_request/4 %% Note that diameter will set Result-Code and Failed-AVPs if %% #diameter_packet.errors is non-null. -handle_request(#diameter_packet{header = H, msg = M, avps = As}, +handle_request(#diameter_packet{header = H, avps = As} + = Pkt, _, - {_Ref, Caps}) -> + {_Ref, Caps}, + #group{encoding = E, + server_decoding = D} + = Grp) -> #diameter_header{end_to_end_id = EI, hop_by_hop_id = HI} = H, @@ -1343,24 +1660,62 @@ handle_request(#diameter_packet{header = H, msg = M, avps = As}, V = EI bsr B, %% assert V = HI bsr B, %% #diameter_caps{origin_state_id = {_,[Id]}} = Caps, - answer(origin(Id), request(M, [H|As], Caps)). + {D,E} = T = origin(Id), %% assert + wrap(T, H, request(to_map(Pkt, Grp), [H|As], Caps)). + +wrap(Id, H, {Tag, Action, Post}) -> + {Tag, wrap(Id, H, Action), Post}; -answer(T, {Tag, Action, Post}) -> - {Tag, answer(T, Action), Post}; -answer(_, {reply, [#diameter_header{} | _]} = T) -> +wrap(_, _, {reply, [#diameter_header{} | _]} = T) -> T; -answer({A,C}, {reply, Ans}) -> - answer(C, {reply, msg(Ans, A, diameter_gen_base_rfc3588)}); -answer(pkt, {reply, Ans}) - when not is_record(Ans, diameter_packet) -> - {reply, #diameter_packet{msg = Ans}}; -answer(_, T) -> + +wrap({_,E}, H, {reply, Ans}) -> + Msg = base_to_nas(msg(Ans, E, diameter_gen_base_rfc3588), H), + {reply, wrap(Msg)}; + +wrap(_, _, T) -> T. +%% Randomly wrap the answer in a diameter_packet. + +wrap(#diameter_packet{} = Pkt) -> + Pkt; + +wrap(Msg) -> + case rand:uniform(2) of + 1 -> #diameter_packet{msg = Msg}; + 2 -> Msg + end. + +%% base_to_nas/2 + +base_to_nas(#diameter_packet{msg = Msg} = Pkt, H) -> + Pkt#diameter_packet{msg = base_to_nas(Msg, H)}; + +base_to_nas(Rec, #diameter_header{application_id = 1}) + when is_tuple(Rec), not ?is_record(Rec, 'diameter_base_answer-message') -> + D = case element(1, Rec) of + diameter_base_accounting_ACA -> + diameter_gen_base_accounting; + _ -> + diameter_gen_base_rfc3588 + end, + [R | Values] = D:'#get-'(Rec), + "diameter_base_" ++ N = ?L(R), + Name = ?A("nas_" ++ if N == "accounting_ACA" -> + "ACA"; + true -> + N + end), + nas4005:'#new-'([Name | Values]); + +base_to_nas(Msg, _) -> + Msg. + %% request/3 %% send_experimental_result -request(#diameter_base_accounting_ACR{'Accounting-Record-Number' = 5}, +request(['ACR' | #{'Accounting-Record-Number' := 5}], [Hdr | Avps], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> @@ -1393,14 +1748,14 @@ request(Msg, _Avps, Caps) -> %% request/2 %% send_nok -request(#diameter_base_accounting_ACR{'Accounting-Record-Number' = 0}, +request(['ACR' | #{'Accounting-Record-Number' := 0}], _) -> {eval_packet, {protocol_error, ?INVALID_AVP_BITS}, [fun log/2, invalid]}; %% send_bad_answer -request(#diameter_base_accounting_ACR{'Session-Id' = SId, - 'Accounting-Record-Type' = RT, - 'Accounting-Record-Number' = 2 = RN}, +request(['ACR' | #{'Session-Id' := SId, + 'Accounting-Record-Type' := RT, + 'Accounting-Record-Number' := 2 = RN}], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> Ans = ['ACA', {'Result-Code', ?SUCCESS}, @@ -1414,9 +1769,9 @@ request(#diameter_base_accounting_ACR{'Session-Id' = SId, msg = Ans}}; %% send_eval -request(#diameter_base_accounting_ACR{'Session-Id' = SId, - 'Accounting-Record-Type' = RT, - 'Accounting-Record-Number' = 3 = RN}, +request(['ACR' | #{'Session-Id' := SId, + 'Accounting-Record-Type' := RT, + 'Accounting-Record-Number' := 3 = RN}], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> Ans = ['ACA', {'Result-Code', ?SUCCESS}, @@ -1428,9 +1783,9 @@ request(#diameter_base_accounting_ACR{'Session-Id' = SId, {eval, {reply, Ans}, {erlang, now, []}}; %% send_ok -request(#diameter_base_accounting_ACR{'Session-Id' = SId, - 'Accounting-Record-Type' = RT, - 'Accounting-Record-Number' = 1 = RN}, +request(['ACR' | #{'Session-Id' := SId, + 'Accounting-Record-Type' := RT, + 'Accounting-Record-Number' := 1 = RN}], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> {reply, ['ACA', {'Result-Code', ?SUCCESS}, @@ -1441,48 +1796,69 @@ request(#diameter_base_accounting_ACR{'Session-Id' = SId, {'Accounting-Record-Number', RN}]}; %% send_protocol_error -request(#diameter_base_accounting_ACR{'Accounting-Record-Number' = 4}, +request(['ACR' | #{'Accounting-Record-Number' := 4}], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> + %% Include a DOIC AVP that will be encoded/decoded because of + %% avp_dictionaries config. + OLR = #{'OC-Sequence-Number' => 1, + 'OC-Report-Type' => 0, %% HOST_REPORT + 'OC-Reduction-Percentage' => [25], + 'OC-Validity-Duration' => [60], + 'AVP' => [{'OC-Supported-Features', []}]}, + %% Include a NAS Failed-AVP AVP that will only be decoded under + %% that application. Encode as 'AVP' since RFC 3588 doesn't list + %% Failed-AVP in the answer-message grammar while RFC 6733 does. + NP = #diameter_avp{data = {nas4005, 'NAS-Port', 44}}, + FR = #diameter_avp{name = 'Firmware-Revision', value = 12}, %% M=0 + AP = #diameter_avp{name = 'Auth-Grace-Period', value = 13}, %% M=1 + Failed = #diameter_avp{data = {diameter_gen_base_rfc3588, + 'Failed-AVP', + [{'AVP', [NP,FR,AP]}]}}, Ans = ['answer-message', {'Result-Code', ?TOO_BUSY}, {'Origin-Host', OH}, - {'Origin-Realm', OR}], + {'Origin-Realm', OR}, + {'AVP', [{'OC-OLR', OLR}, Failed]}], {reply, Ans}; -request(#diameter_base_ASR{'Session-Id' = SId, - 'AVP' = Avps}, +%% send_proxy_info +request(['ASR' | #{'Proxy-Info' := _}], + _) -> + {protocol_error, 3999}; + +request(['ASR' | #{'Session-Id' := SId} = Avps], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> {reply, ['ASA', {'Result-Code', ?SUCCESS}, {'Session-Id', SId}, {'Origin-Host', OH}, {'Origin-Realm', OR}, - {'AVP', Avps}]}; + {'AVP', maps:get('AVP', Avps, [])}]}; %% send_invalid_reject -request(#diameter_base_STR{'Termination-Cause' = ?USER_MOVED}, +request(['STR' | #{'Termination-Cause' := ?USER_MOVED}], _Caps) -> {protocol_error, ?TOO_BUSY}; %% send_noreply -request(#diameter_base_STR{'Termination-Cause' = T}, +request(['STR' | #{'Termination-Cause' := T}], _Caps) when T /= ?LOGOUT -> discard; %% send_destination_5 -request(#diameter_base_STR{'Destination-Realm' = R}, +request(['STR' | #{'Destination-Realm' := R}], #diameter_caps{origin_realm = {OR, _}}) when R /= undefined, R /= OR -> {protocol_error, ?REALM_NOT_SERVED}; %% send_destination_6 -request(#diameter_base_STR{'Destination-Host' = [H]}, +request(['STR' | #{'Destination-Host' := [H]}], #diameter_caps{origin_host = {OH, _}}) when H /= OH -> {protocol_error, ?UNABLE_TO_DELIVER}; -request(#diameter_base_STR{'Session-Id' = SId}, +request(['STR' | #{'Session-Id' := SId}], #diameter_caps{origin_host = {OH, _}, origin_realm = {OR, _}}) -> {reply, ['STA', {'Result-Code', ?SUCCESS}, @@ -1491,7 +1867,7 @@ request(#diameter_base_STR{'Session-Id' = SId}, {'Origin-Realm', OR}]}; %% send_error/send_timeout -request(#diameter_base_RAR{}, _Caps) -> +request(['RAR' | #{}], _Caps) -> receive after 2000 -> {protocol_error, ?TOO_BUSY} end. %% message/3 @@ -1505,8 +1881,8 @@ message(Dir, #diameter_packet{bin = Bin}, N) -> message(Dir, Bin, N); %% incoming request -message(recv, <<_:32, 1, _/bits>> = Bin, N) -> - [Bin, 1 < N, fun ?MODULE:message/3, N-1]; +message(recv, <<_:32, 1:1, _/bits>> = Bin, N) -> + [Bin, N < 16, fun ?MODULE:message/3, N+1]; %% incoming answer message(recv, Bin, _) -> @@ -1517,9 +1893,35 @@ message(send, Bin, _) -> [Bin]; %% sent request -message(ack, <<_:32, 1, _/bits>>, _) -> +message(ack, <<_:32, 1:1, _/bits>>, _) -> []; %% sent answer or discarded request message(ack, _, N) -> - [0 =< N, fun ?MODULE:message/3, N+1]. + [N =< 16, fun ?MODULE:message/3, N-1]. + +%% ------------------------------------------------------------------------ + +compile_and_load() -> + try + Path = hd([P || H <- [[here(), ".."], [code:lib_dir(diameter)]], + P <- [filename:join(H ++ ["examples", + "dict", + "rfc4005_nas.dia"])], + {ok, _} <- [file:read_file_info(P)]]), + {ok, [Forms]} + = diameter_make:codec(Path, [return, + forms, + {name, "nas4005"}, + {prefix, "nas"}, + {inherits, "common/diameter_gen_base_rfc3588"}]), + {ok, nas4005, Bin, []} = compile:forms(Forms, [debug_info, return]), + {module, nas4005} = code:load_binary(nas4005, "nas4005", Bin), + true + catch + E:R -> + {E, R, erlang:get_stacktrace()} + end. + +here() -> + filename:dirname(code:which(?MODULE)). diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl index 9d981d0a2b..284d2b9566 100644 --- a/lib/diameter/test/diameter_transport_SUITE.erl +++ b/lib/diameter/test/diameter_transport_SUITE.erl @@ -349,35 +349,40 @@ rand_bytes(N) -> %% start_connect/3 start_connect(Prot, PortNr, Ref) -> - {ok, TPid, [?ADDR]} = start_connect(Prot, - {connect, Ref}, - ?SVC([]), - [{raddr, ?ADDR}, - {rport, PortNr}, - {ip, ?ADDR}, - {port, 0}]), - ?RECV(?TMSG({TPid, connected, _})), + {ok, TPid} = start_connect(Prot, + {connect, Ref}, + ?SVC([]), + [{raddr, ?ADDR}, + {rport, PortNr}, + {ip, ?ADDR}, + {port, 0}]), + connected(Prot, TPid), TPid. +connected(sctp, TPid) -> + ?RECV(?TMSG({TPid, connected, _})); +connected(tcp, TPid) -> + ?RECV(?TMSG({TPid, connected, _, [?ADDR]})). + start_connect(sctp, T, Svc, Opts) -> - diameter_sctp:start(T, Svc, [{sctp_initmsg, ?SCTP_INIT} | Opts]); + {ok, TPid, [?ADDR]} + = diameter_sctp:start(T, Svc, [{sctp_initmsg, ?SCTP_INIT} | Opts]), + {ok, TPid}; start_connect(tcp, T, Svc, Opts) -> diameter_tcp:start(T, Svc, Opts). %% start_accept/2 start_accept(Prot, Ref) -> - {Mod, Opts} = tmod(Prot), - {ok, TPid, [?ADDR]} = Mod:start({accept, Ref}, - ?SVC([?ADDR]), - [{port, 0} | Opts]), + {ok, TPid, [?ADDR]} + = start_accept(Prot, {accept, Ref}, ?SVC([?ADDR]), [{port, 0}]), ?RECV(?TMSG({TPid, connected})), TPid. -tmod(sctp) -> - {diameter_sctp, [{sctp_initmsg, ?SCTP_INIT}]}; -tmod(tcp) -> - {diameter_tcp, []}. +start_accept(sctp, T, Svc, Opts) -> + diameter_sctp:start(T, Svc, [{sctp_initmsg, ?SCTP_INIT} | Opts]); +start_accept(tcp, T, Svc, Opts) -> + diameter_tcp:start(T, Svc, Opts). %% =========================================================================== diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl index 03f79096ac..d249b0e4fa 100644 --- a/lib/diameter/test/diameter_util.erl +++ b/lib/diameter/test/diameter_util.erl @@ -32,7 +32,8 @@ foldl/3, scramble/1, unique_string/0, - have_sctp/0]). + have_sctp/0, + eprof/1]). %% diameter-specific -export([lport/2, @@ -48,6 +49,16 @@ -define(L, atom_to_list). +%% --------------------------------------------------------------------------- + +eprof(start) -> + eprof:start(), + eprof:start_profiling([self()]); + +eprof(stop) -> + eprof:stop_profiling(), + eprof:analyze(), + eprof:stop(). %% --------------------------------------------------------------------------- %% name/2 diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 66ec6cabd8..29e4b22632 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -408,7 +408,7 @@ <c>{self, once}</c>, the first message has an extra element, that is, <c>{http, {RequestId, stream_start, Headers, Pid}}</c>. This is the process id to be used as an argument to - <c>http:stream_next/1</c> to trigger the next message to be sent to + <c>httpc:stream_next/1</c> to trigger the next message to be sent to the calling process.</p> <p>Notice that chunked encoding can add headers so that there are more headers in the <c>stream_end</c> diff --git a/lib/inets/src/http_server/mod_disk_log.erl b/lib/inets/src/http_server/mod_disk_log.erl index 3be5f2dd74..2023546f01 100644 --- a/lib/inets/src/http_server/mod_disk_log.erl +++ b/lib/inets/src/http_server/mod_disk_log.erl @@ -363,17 +363,21 @@ create_disk_log(Filename, MaxBytes, MaxFiles, ConfigList) -> %%---------------------------------------------------------------------- open(Filename, MaxBytes, MaxFiles, internal) -> - Opts = [{format, internal}, {repair, truncate}], - open1(Filename, MaxBytes, MaxFiles, Opts); + Opt0 = {format, internal}, + Opts1 = [Opt0, {repair, true}], + Opts2 = [Opt0, {repair, truncate}], + open1(Filename, MaxBytes, MaxFiles, Opts1, Opts2); open(Filename, MaxBytes, MaxFiles, _) -> Opts = [{format, external}], - open1(Filename, MaxBytes, MaxFiles, Opts). + open1(Filename, MaxBytes, MaxFiles, Opts, Opts). -open1(Filename, MaxBytes, MaxFiles, Opts0) -> - Opts1 = [{name, Filename}, {file, Filename}, {type, wrap}] ++ Opts0, - case open2(Opts1, {MaxBytes, MaxFiles}) of +open1(Filename, MaxBytes, MaxFiles, Opts1, Opts2) -> + Opts0 = [{name, Filename}, {file, Filename}, {type, wrap}], + case open2(Opts0 ++ Opts1, Opts0 ++ Opts2, {MaxBytes, MaxFiles}) of {ok, LogDB} -> {ok, LogDB}; + {repaired, LogDB, {recovered, _}, {badbytes, _}} -> + {ok, LogDB}; {error, Reason} -> {error, ?NICE("Can't create " ++ Filename ++ @@ -382,11 +386,16 @@ open1(Filename, MaxBytes, MaxFiles, Opts0) -> {error, ?NICE("Can't create "++Filename)} end. -open2(Opts, Size) -> - case disk_log:open(Opts) of +open2(Opts1, Opts2, Size) -> + case disk_log:open(Opts1) of {error, {badarg, size}} -> %% File did not exist, add the size option and try again - disk_log:open([{size, Size} | Opts]); + disk_log:open([{size, Size} | Opts1]); + {error, {Reason, _}} when + Reason == not_a_log_file; + Reason == invalid_index_file -> + %% File was corrupt, add the truncate option and try again + disk_log:open([{size, Size} | Opts2]); Else -> Else end. diff --git a/lib/inets/src/http_server/mod_log.erl b/lib/inets/src/http_server/mod_log.erl index ad7e9713d9..ec570504be 100644 --- a/lib/inets/src/http_server/mod_log.erl +++ b/lib/inets/src/http_server/mod_log.erl @@ -105,8 +105,8 @@ do(Info) -> Code = proplists:get_value(code,Head,unknown), transfer_log(Info, "-", AuthUser, Date, Code, Size), {proceed, Info#mod.data}; - {_StatusCode, Response} -> - transfer_log(Info,"-",AuthUser,Date,200, + {StatusCode, Response} -> + transfer_log(Info, "-", AuthUser, Date, StatusCode, httpd_util:flatlength(Response)), {proceed,Info#mod.data}; undefined -> diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 055b847319..b4f0f2aa7d 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -73,6 +73,7 @@ all() -> {group, http_reload}, {group, https_reload}, {group, http_mime_types}, + {group, http_logging}, mime_types_format ]. @@ -96,6 +97,7 @@ groups() -> {https_htaccess, [], [{group, htaccess}]}, {http_security, [], [{group, security}]}, {https_security, [], [{group, security}]}, + {http_logging, [], [{group, logging}]}, {http_reload, [], [{group, reload}]}, {https_reload, [], [{group, reload}]}, {http_mime_types, [], [alias_1_1, alias_1_0, alias_0_9]}, @@ -119,6 +121,8 @@ groups() -> ]}, {htaccess, [], [htaccess_1_1, htaccess_1_0, htaccess_0_9]}, {security, [], [security_1_1, security_1_0]}, %% Skip 0.9 as causes timing issus in test code + {logging, [], [disk_log_internal, disk_log_exists, + disk_log_bad_size, disk_log_bad_file]}, {http_1_1, [], [host, chunked, expect, cgi, cgi_chunked_encoding_test, trace, range, if_modified_since, mod_esi_chunk_timeout, @@ -254,6 +258,11 @@ init_per_group(auth_api_dets, Config) -> init_per_group(auth_api_mnesia, Config) -> start_mnesia(proplists:get_value(node, Config)), [{auth_prefix, "mnesia_"} | Config]; +init_per_group(http_logging, Config) -> + Config1 = [{http_version, "HTTP/1.1"} | Config], + ServerRoot = proplists:get_value(server_root, Config1), + Path = ServerRoot ++ "/httpd_log_transfer", + [{transfer_log, Path} | Config1]; init_per_group(_, Config) -> Config. @@ -310,10 +319,60 @@ init_per_testcase(range, Config) -> create_range_data(DocRoot), dbg(range, Config, init); +init_per_testcase(disk_log_internal, Config0) -> + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + +init_per_testcase(disk_log_exists, Config0) -> + ServerRoot = proplists:get_value(server_root, Config0), + Filename = ServerRoot ++ "/httpd_log_transfer", + {ok, Log} = disk_log:open([{name, Filename}, {file, Filename}, + {repair, truncate}, {format, internal}, + {type, wrap}, {size, {1048576, 5}}]), + ok = disk_log:log(Log, {bogus, node(), self()}), + ok = disk_log:close(Log), + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + +init_per_testcase(disk_log_bad_size, Config0) -> + ServerRoot = proplists:get_value(server_root, Config0), + Filename = ServerRoot ++ "/httpd_log_transfer", + {ok, Log} = disk_log:open([{name, Filename}, {file, Filename}, + {repair, truncate}, {format, internal}, + {type, wrap}, {size, {1048576, 5}}]), + ok = disk_log:log(Log, {bogus, node(), self()}), + ok = disk_log:close(Log), + ok = file:delete(Filename ++ ".siz"), + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + +init_per_testcase(disk_log_bad_file, Config0) -> + ServerRoot = proplists:get_value(server_root, Config0), + Filename = ServerRoot ++ "/httpd_log_transfer", + ok = file:write_file(Filename ++ ".1", <<>>), + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + init_per_testcase(Case, Config) -> ct:timetrap({seconds, 20}), dbg(Case, Config, init). +end_per_testcase(Case, Config) when + Case == disk_log_internal; + Case == disk_log_exists; + Case == disk_log_bad_size; + Case == disk_log_bad_file -> + inets:stop(), + dbg(Case, Config, 'end'); + end_per_testcase(Case, Config) -> dbg(Case, Config, 'end'). @@ -1257,6 +1316,63 @@ security(Config) -> true = unblock_user(Node, "two", Port, OpenDir). %%------------------------------------------------------------------------- + +disk_log_internal() -> + ["Test mod_disk_log"]. + +disk_log_internal(Config) -> + Version = proplists:get_value(http_version, Config), + Request = "GET /" ++ integer_to_list(rand:uniform(1000000)) ++ " ", + ok = http_status(Request, Config, [{statuscode, 404}]), + Log = proplists:get_value(transfer_log, Config), + Match = list_to_binary(Request ++ Version), + disk_log_internal1(Log, Match, disk_log:chunk(Log, start)). +disk_log_internal1(_, _, eof) -> + ct:fail(eof); +disk_log_internal1(Log, Match, {Cont, [H | T]}) -> + case binary:match(H, Match) of + nomatch -> + disk_log_internal1(Log, Match, {Cont, T}); + _ -> + ok + end; +disk_log_internal1(Log, Match, {Cont, []}) -> + disk_log_internal1(Log, Match, disk_log:chunk(Log, Cont)). + +disk_log_exists() -> + ["Test mod_disk_log with existing logs"]. + +disk_log_exists(Config) -> + Log = proplists:get_value(transfer_log, Config), + Self = self(), + Node = node(), + Log = proplists:get_value(transfer_log, Config), + {_, [{bogus, Node, Self} | _]} = disk_log:chunk(Log, start). + +disk_log_bad_size() -> + ["Test mod_disk_log with existing log, missing .siz"]. + +disk_log_bad_size(Config) -> + Log = proplists:get_value(transfer_log, Config), + Self = self(), + Node = node(), + Log = proplists:get_value(transfer_log, Config), + {_, [{bogus, Node, Self} | _]} = disk_log:chunk(Log, start). + +disk_log_bad_file() -> + ["Test mod_disk_log with bad file"]. + +disk_log_bad_file(Config) -> + Log = proplists:get_value(transfer_log, Config), + Version = proplists:get_value(http_version, Config), + Request = "GET /" ++ integer_to_list(rand:uniform(1000000)) ++ " ", + ok = http_status(Request, Config, [{statuscode, 404}]), + Log = proplists:get_value(transfer_log, Config), + Match = list_to_binary(Request ++ Version), + {_, [H | _]} = disk_log:chunk(Log, start), + {_, _} = binary:match(H, Match). + +%%------------------------------------------------------------------------- non_disturbing_reconfiger_dies(Config) when is_list(Config) -> do_reconfiger_dies([{http_version, "HTTP/1.1"} | Config], non_disturbing). disturbing_reconfiger_dies(Config) when is_list(Config) -> @@ -1567,6 +1683,7 @@ start_apps(Group) when Group == http_basic; Group == http_auth_api_mnesia; Group == http_htaccess; Group == http_security; + Group == http_logging; Group == http_reload; Group == http_mime_types-> inets_test_lib:start_apps([inets]). @@ -1662,6 +1779,8 @@ server_config(http_security, Config) -> server_config(https_security, Config) -> ServerRoot = proplists:get_value(server_root, Config), tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(https, Config); +server_config(http_logging, Config) -> + log_conf() ++ server_config(http, Config); server_config(http_mime_types, Config0) -> Config1 = basic_conf() ++ server_config(http, Config0), ServerRoot = proplists:get_value(server_root, Config0), @@ -1863,6 +1982,16 @@ mod_security_conf(SecFile, Dir) -> {path, Dir} %% This is should not be needed, but is atm, awful design! ]. +log_conf() -> + [{modules, [mod_alias, mod_dir, mod_get, mod_head, mod_disk_log]}, + {transfer_disk_log, "httpd_log_transfer"}, + {security_disk_log, "httpd_log_security"}, + {error_disk_log, "httpd_log_error"}, + {transfer_disk_log_size, {1048576, 5}}, + {error_disk_log_size, {1048576, 5}}, + {error_disk_log_size, {1048576, 5}}, + {security_disk_log_size, {1048576, 5}}, + {disk_log_format, internal}]. http_status(Request, Config, Expected) -> Version = proplists:get_value(http_version, Config), diff --git a/lib/kernel/doc/src/disk_log.xml b/lib/kernel/doc/src/disk_log.xml index 1be28adfb8..884cb32c0c 100644 --- a/lib/kernel/doc/src/disk_log.xml +++ b/lib/kernel/doc/src/disk_log.xml @@ -972,7 +972,7 @@ <item> <p>Specifies if messages will be sent to <c>error_logger</c> on recoverable errors with - the log files. Defaults to <c>true</c>.</p> + the log files. Defaults to <c>false</c>.</p> </item> </taglist> <p><c>open/1</c> returns <c>{ok, <anno>Log</anno>}</c> if the diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index b71e8a1e5d..169a76463b 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -222,11 +222,18 @@ fe80::204:acff:fe17:bf38 <name name="get_rc" arity="0"/> <fsummary>Return a list of IP configuration parameters.</fsummary> <desc> - <p>Returns the state of the <c>Inet</c> configuration database in + <p> + Returns the state of the <c>Inet</c> configuration database in form of a list of recorded configuration parameters. For more information, see <seealso marker="erts:inet_cfg">ERTS User's Guide: Inet Configuration</seealso>. - Only parameters with other than default values are returned.</p> + </p> + <p> + Only actual parameters with other than default values + are returned, for example not directives that specify + other sources for configuration parameters nor + directives that clear parameters. + </p> </desc> </func> diff --git a/lib/kernel/doc/src/inet_res.xml b/lib/kernel/doc/src/inet_res.xml index 4ada4203c0..3454e3c6f9 100644 --- a/lib/kernel/doc/src/inet_res.xml +++ b/lib/kernel/doc/src/inet_res.xml @@ -130,7 +130,7 @@ dns_header() = DnsHeader inet_dns:header(DnsHeader) -> [ {id, integer()} | {qr, boolean()} - | {opcode, 'query' | iquery | status | integer()} + | {opcode, query | iquery | status | integer()} | {aa, boolean()} | {tc, boolean()} | {rd, boolean()} diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index e1cf45109d..9cd03ffcad 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 5.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>The documentation for the 'quiet' option in + disk_log:open/1 had an incorrect default value.</p> + <p> + Own Id: OTP-14498</p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 5.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/examples/Makefile b/lib/kernel/examples/Makefile index 26ec58f571..f86e662838 100644 --- a/lib/kernel/examples/Makefile +++ b/lib/kernel/examples/Makefile @@ -45,7 +45,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/kernel-$(KERNEL_VSN)/examples # Pack and install the complete directory structure from # here (CWD) and down, for all examples. -EXAMPLES = uds_dist +EXAMPLES = uds_dist gen_tcp_dist release_spec: $(INSTALL_DIR) "$(RELSYSDIR)" diff --git a/lib/kernel/examples/gen_tcp_dist/Makefile b/lib/kernel/examples/gen_tcp_dist/Makefile new file mode 100644 index 0000000000..65513a1729 --- /dev/null +++ b/lib/kernel/examples/gen_tcp_dist/Makefile @@ -0,0 +1,20 @@ +RM=rm -f +CP=cp +EBIN=ebin +ERLC=erlc +# Works if building in open source source tree +KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include +ERLCFLAGS+= -W -I$(KERNEL_INCLUDE) + +MODULES=gen_tcp_dist + +TARGET_FILES=$(MODULES:%=$(EBIN)/%.beam) + +opt: $(TARGET_FILES) + +$(EBIN)/%.beam: src/%.erl + $(ERLC) $(ERLCFLAGS) -o$(EBIN) $< + +clean: + $(RM) $(TARGET_FILES) + diff --git a/lib/kernel/examples/gen_tcp_dist/ebin/.gitignore b/lib/kernel/examples/gen_tcp_dist/ebin/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/kernel/examples/gen_tcp_dist/ebin/.gitignore diff --git a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl new file mode 100644 index 0000000000..98554ed805 --- /dev/null +++ b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl @@ -0,0 +1,781 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(gen_tcp_dist). + +%% +%% This is an example of how to plug in an arbitrary distribution +%% carrier for Erlang using distribution processes. +%% +%% This example uses gen_tcp for transportation of data, but +%% you can use whatever underlying protocol you want as long +%% as your implementation reliably delivers data chunks to the +%% receiving VM in the order they were sent from the sending +%% VM. +%% +%% This code is a rewrite of the lib/kernel/src/inet_tcp_dist.erl +%% distribution impementation for TCP used by default. That +%% implementation use distribution ports instead of distribution +%% processes and is more efficient compared to this implementation. +%% This since this implementation more or less gets the +%% distribution processes in between the VM and the ports without +%% any gain specific gain. +%% + +-export([listen/1, accept/1, accept_connection/5, + setup/5, close/1, select/1, is_node_name/1]). + +%% Optional +-export([setopts/2, getopts/2]). + +%% internal exports + +-export([dist_cntrlr_setup/1, dist_cntrlr_input_setup/3, + dist_cntrlr_tick_handler/1]). + +-export([accept_loop/2,do_accept/6,do_setup/6]). + +-import(error_logger,[error_msg/2]). + +-include("net_address.hrl"). + +-include("dist.hrl"). +-include("dist_util.hrl"). + +%% ------------------------------------------------------------ +%% Select this protocol based on node name +%% select(Node) => Bool +%% ------------------------------------------------------------ + +select(Node) -> + case split_node(atom_to_list(Node), $@, []) of + [_, Host] -> + case inet:getaddr(Host, inet) of + {ok,_} -> true; + _ -> false + end; + _ -> false + end. + +%% ------------------------------------------------------------ +%% Create the listen socket, i.e. the port that this erlang +%% node is accessible through. +%% ------------------------------------------------------------ + +listen(Name) -> + case do_listen([binary, {active, false}, {packet,2}, {reuseaddr, true}]) of + {ok, Socket} -> + TcpAddress = get_tcp_address(Socket), + {_,Port} = TcpAddress#net_address.address, + ErlEpmd = net_kernel:epmd_module(), + case ErlEpmd:register_node(Name, Port) of + {ok, Creation} -> + {ok, {Socket, TcpAddress, Creation}}; + Error -> + Error + end; + Error -> + Error + end. + +do_listen(Options) -> + {First,Last} = case application:get_env(kernel,inet_dist_listen_min) of + {ok,N} when is_integer(N) -> + case application:get_env(kernel, + inet_dist_listen_max) of + {ok,M} when is_integer(M) -> + {N,M}; + _ -> + {N,N} + end; + _ -> + {0,0} + end, + do_listen(First, Last, listen_options([{backlog,128}|Options])). + +do_listen(First,Last,_) when First > Last -> + {error,eaddrinuse}; +do_listen(First,Last,Options) -> + case gen_tcp:listen(First, Options) of + {error, eaddrinuse} -> + do_listen(First+1,Last,Options); + Other -> + Other + end. + +listen_options(Opts0) -> + Opts1 = + case application:get_env(kernel, inet_dist_use_interface) of + {ok, Ip} -> + [{ip, Ip} | Opts0]; + _ -> + Opts0 + end, + case application:get_env(kernel, inet_dist_listen_options) of + {ok,ListenOpts} -> + ListenOpts ++ Opts1; + _ -> + Opts1 + end. + + +%% ------------------------------------------------------------ +%% Accepts new connection attempts from other Erlang nodes. +%% ------------------------------------------------------------ + +accept(Listen) -> + spawn_opt(?MODULE, accept_loop, [self(), Listen], [link, {priority, max}]). + +accept_loop(Kernel, Listen) -> + ?trace("~p~n",[{?MODULE, accept_loop, self()}]), + case gen_tcp:accept(Listen) of + {ok, Socket} -> + DistCtrl = spawn_dist_cntrlr(Socket), + ?trace("~p~n",[{?MODULE, accept_loop, accepted, Socket, DistCtrl, self()}]), + flush_controller(DistCtrl, Socket), + gen_tcp:controlling_process(Socket, DistCtrl), + flush_controller(DistCtrl, Socket), + Kernel ! {accept,self(),DistCtrl,inet,tcp}, + receive + {Kernel, controller, Pid} -> + call_ctrlr(DistCtrl, {supervisor, Pid}), + Pid ! {self(), controller}; + {Kernel, unsupported_protocol} -> + exit(unsupported_protocol) + end, + accept_loop(Kernel, Listen); + Error -> + exit(Error) + end. + +flush_controller(Pid, Socket) -> + receive + {tcp, Socket, Data} -> + Pid ! {tcp, Socket, Data}, + flush_controller(Pid, Socket); + {tcp_closed, Socket} -> + Pid ! {tcp_closed, Socket}, + flush_controller(Pid, Socket) + after 0 -> + ok + end. + +%% ------------------------------------------------------------ +%% Accepts a new connection attempt from another Erlang node. +%% Performs the handshake with the other side. +%% ------------------------------------------------------------ + +accept_connection(AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) -> + spawn_opt(?MODULE, do_accept, + [self(), AcceptPid, DistCtrl, MyNode, Allowed, SetupTime], + [link, {priority, max}]). + +do_accept(Kernel, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) -> + ?trace("~p~n",[{?MODULE, do_accept, self(), MyNode}]), + receive + {AcceptPid, controller} -> + Timer = dist_util:start_timer(SetupTime), + case check_ip(DistCtrl) of + true -> + HSData0 = hs_data_common(DistCtrl), + HSData = HSData0#hs_data{kernel_pid = Kernel, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + this_flags = 0, + allowed = Allowed}, + dist_util:handshake_other_started(HSData); + {false,IP} -> + error_msg("** Connection attempt from " + "disallowed IP ~w ** ~n", [IP]), + ?shutdown(no_node) + end + end. + +%% we may not always want the nodelay behaviour +%% for performance reasons + +nodelay() -> + case application:get_env(kernel, dist_nodelay) of + undefined -> + {nodelay, true}; + {ok, true} -> + {nodelay, true}; + {ok, false} -> + {nodelay, false}; + _ -> + {nodelay, true} + end. + +%% ------------------------------------------------------------ +%% Setup a new connection to another Erlang node. +%% Performs the handshake with the other side. +%% ------------------------------------------------------------ + +setup(Node, Type, MyNode, LongOrShortNames,SetupTime) -> + spawn_opt(?MODULE, do_setup, + [self(), Node, Type, MyNode, LongOrShortNames, SetupTime], + [link, {priority, max}]). + +do_setup(Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) -> + ?trace("~p~n",[{?MODULE, do_setup, self(), Node}]), + [Name, Address] = splitnode(Node, LongOrShortNames), + case inet:getaddr(Address, inet) of + {ok, Ip} -> + Timer = dist_util:start_timer(SetupTime), + ErlEpmd = net_kernel:epmd_module(), + case ErlEpmd:port_please(Name, Ip) of + {port, TcpPort, Version} -> + ?trace("port_please(~p) -> version ~p~n", + [Node,Version]), + dist_util:reset_timer(Timer), + case + gen_tcp:connect( + Ip, TcpPort, + connect_options([binary, {active, false}, {packet, 2}])) + of + {ok, Socket} -> + DistCtrl = spawn_dist_cntrlr(Socket), + call_ctrlr(DistCtrl, {supervisor, self()}), + flush_controller(DistCtrl, Socket), + gen_tcp:controlling_process(Socket, DistCtrl), + flush_controller(DistCtrl, Socket), + HSData0 = hs_data_common(DistCtrl), + HSData = HSData0#hs_data{kernel_pid = Kernel, + other_node = Node, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + this_flags = 0, + other_version = Version, + request_type = Type}, + dist_util:handshake_we_started(HSData); + _ -> + %% Other Node may have closed since + %% port_please ! + ?trace("other node (~p) " + "closed since port_please.~n", + [Node]), + ?shutdown(Node) + end; + _ -> + ?trace("port_please (~p) " + "failed.~n", [Node]), + ?shutdown(Node) + end; + _Other -> + ?trace("inet_getaddr(~p) " + "failed (~p).~n", [Node,_Other]), + ?shutdown(Node) + end. + +connect_options(Opts) -> + case application:get_env(kernel, inet_dist_connect_options) of + {ok,ConnectOpts} -> + ConnectOpts ++ Opts; + _ -> + Opts + end. + +%% +%% Close a socket. +%% +close(Listen) -> + gen_tcp:close(Listen). + + +%% If Node is illegal terminate the connection setup!! +splitnode(Node, LongOrShortNames) -> + case split_node(atom_to_list(Node), $@, []) of + [Name|Tail] when Tail =/= [] -> + Host = lists:append(Tail), + case split_node(Host, $., []) of + [_] when LongOrShortNames =:= longnames -> + case inet:parse_address(Host) of + {ok, _} -> + [Name, Host]; + _ -> + error_msg("** System running to use " + "fully qualified " + "hostnames **~n" + "** Hostname ~ts is illegal **~n", + [Host]), + ?shutdown(Node) + end; + L when length(L) > 1, LongOrShortNames =:= shortnames -> + error_msg("** System NOT running to use fully qualified " + "hostnames **~n" + "** Hostname ~ts is illegal **~n", + [Host]), + ?shutdown(Node); + _ -> + [Name, Host] + end; + [_] -> + error_msg("** Nodename ~p illegal, no '@' character **~n", + [Node]), + ?shutdown(Node); + _ -> + error_msg("** Nodename ~p illegal **~n", [Node]), + ?shutdown(Node) + end. + +split_node([Chr|T], Chr, Ack) -> [lists:reverse(Ack)|split_node(T, Chr, [])]; +split_node([H|T], Chr, Ack) -> split_node(T, Chr, [H|Ack]); +split_node([], _, Ack) -> [lists:reverse(Ack)]. + +%% ------------------------------------------------------------ +%% Fetch local information about a Socket. +%% ------------------------------------------------------------ +get_tcp_address(Socket) -> + {ok, Address} = inet:sockname(Socket), + {ok, Host} = inet:gethostname(), + #net_address { + address = Address, + host = Host, + protocol = tcp, + family = inet + }. + +%% ------------------------------------------------------------ +%% Do only accept new connection attempts from nodes at our +%% own LAN, if the check_ip environment parameter is true. +%% ------------------------------------------------------------ +check_ip(DistCtrl) -> + case application:get_env(check_ip) of + {ok, true} -> + case get_ifs(DistCtrl) of + {ok, IFs, IP} -> + check_ip(IFs, IP); + _ -> + ?shutdown(no_node) + end; + _ -> + true + end. + +get_ifs(DistCtrl) -> + Socket = call_ctrlr(DistCtrl, socket), + case inet:peername(Socket) of + {ok, {IP, _}} -> + case inet:getif(Socket) of + {ok, IFs} -> {ok, IFs, IP}; + Error -> Error + end; + Error -> + Error + end. + +check_ip([{OwnIP, _, Netmask}|IFs], PeerIP) -> + case {inet_tcp:mask(Netmask, PeerIP), inet_tcp:mask(Netmask, OwnIP)} of + {M, M} -> true; + _ -> check_ip(IFs, PeerIP) + end; +check_ip([], PeerIP) -> + {false, PeerIP}. + +is_node_name(Node) when is_atom(Node) -> + case split_node(atom_to_list(Node), $@, []) of + [_, _Host] -> true; + _ -> false + end; +is_node_name(_Node) -> + false. + +hs_data_common(DistCtrl) -> + TickHandler = call_ctrlr(DistCtrl, tick_handler), + Socket = call_ctrlr(DistCtrl, socket), + #hs_data{f_send = send_fun(), + f_recv = recv_fun(), + f_setopts_pre_nodeup = setopts_pre_nodeup_fun(), + f_setopts_post_nodeup = setopts_post_nodeup_fun(), + f_getll = getll_fun(), + f_handshake_complete = handshake_complete_fun(), + f_address = address_fun(), + mf_setopts = setopts_fun(DistCtrl, Socket), + mf_getopts = getopts_fun(DistCtrl, Socket), + mf_getstat = getstat_fun(DistCtrl, Socket), + mf_tick = tick_fun(DistCtrl, TickHandler)}. + +%%% ------------------------------------------------------------ +%%% Distribution controller processes +%%% ------------------------------------------------------------ + +%% +%% There will be five parties working together when the +%% connection is up: +%% - The gen_tcp socket. Providing a tcp/ip connection +%% to the other node. +%% - The output handler. It will dispatch all outgoing +%% traffic from the VM to the gen_tcp socket. This +%% process is registered as distribution controller +%% for this channel with the VM. +%% - The input handler. It will dispatch all incoming +%% traffic from the gen_tcp socket to the VM. This +%% process is also the socket owner and receives +%% incoming traffic using active-N. +%% - The tick handler. Dispatches asynchronous tick +%% requests to the socket. It executes on max priority +%% since it is important to get ticks through to the +%% other end. +%% - The channel supervisor (provided by dist_util). It +%% monitors traffic. Issue tick requests to the tick +%% handler when no outgoing traffic is seen and bring +%% the connection down if no incoming traffic is seen. +%% This process also executes on max priority. +%% +%% These parties are linked togheter so should one +%% of them fail, all of them are terminated and the +%% connection is taken down. +%% + +%% In order to avoid issues with lingering signal binaries +%% we enable off-heap message queue data as well as fullsweep +%% after 0. The fullsweeps will be cheap since we have more +%% or less no live data. +-define(DIST_CNTRL_COMMON_SPAWN_OPTS, + [{message_queue_data, off_heap}, + {fullsweep_after, 0}]). + +tick_fun(DistCtrl, TickHandler) -> + fun (Ctrl) when Ctrl == DistCtrl -> + TickHandler ! tick + end. + +getstat_fun(DistCtrl, Socket) -> + fun (Ctrl) when Ctrl == DistCtrl -> + case inet:getstat(Socket, [recv_cnt, send_cnt, send_pend]) of + {ok, Stat} -> + split_stat(Stat,0,0,0); + Error -> + Error + end + end. + +split_stat([{recv_cnt, R}|Stat], _, W, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_cnt, W}|Stat], R, _, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_pend, P}|Stat], R, W, _) -> + split_stat(Stat, R, W, P); +split_stat([], R, W, P) -> + {ok, R, W, P}. + +setopts_fun(DistCtrl, Socket) -> + fun (Ctrl, Opts) when Ctrl == DistCtrl -> + setopts(Socket, Opts) + end. + +getopts_fun(DistCtrl, Socket) -> + fun (Ctrl, Opts) when Ctrl == DistCtrl -> + getopts(Socket, Opts) + end. + +setopts(S, Opts) -> + case [Opt || {K,_}=Opt <- Opts, + K =:= active orelse K =:= deliver orelse K =:= packet] of + [] -> inet:setopts(S,Opts); + Opts1 -> {error, {badopts,Opts1}} + end. + +getopts(S, Opts) -> + inet:getopts(S, Opts). + +send_fun() -> + fun (Ctrlr, Packet) -> + call_ctrlr(Ctrlr, {send, Packet}) + end. + +recv_fun() -> + fun (Ctrlr, Length, Timeout) -> + case call_ctrlr(Ctrlr, {recv, Length, Timeout}) of + {ok, Bin} when is_binary(Bin) -> + {ok, binary_to_list(Bin)}; + Other -> + Other + end + end. + +getll_fun() -> + fun (Ctrlr) -> + call_ctrlr(Ctrlr, getll) + end. + +address_fun() -> + fun (Ctrlr, Node) -> + case call_ctrlr(Ctrlr, {address, Node}) of + {error, no_node} -> %% No '@' or more than one '@' in node name. + ?shutdown(no_node); + Res -> + Res + end + end. + +setopts_pre_nodeup_fun() -> + fun (Ctrlr) -> + call_ctrlr(Ctrlr, pre_nodeup) + end. + +setopts_post_nodeup_fun() -> + fun (Ctrlr) -> + call_ctrlr(Ctrlr, post_nodeup) + end. + +handshake_complete_fun() -> + fun (Ctrlr, Node, DHandle) -> + call_ctrlr(Ctrlr, {handshake_complete, Node, DHandle}) + end. + +call_ctrlr(Ctrlr, Msg) -> + Ref = erlang:monitor(process, Ctrlr), + Ctrlr ! {Ref, self(), Msg}, + receive + {Ref, Res} -> + erlang:demonitor(Ref, [flush]), + Res; + {'DOWN', Ref, process, Ctrlr, Reason} -> + exit({dist_controller_exit, Reason}) + end. + +%% +%% The tick handler process writes a tick to the +%% socket when it receives a 'tick' message from +%% the connection supervisor. +%% +%% We are not allowed to block the connection +%% superviser when writing a tick and we also want +%% the tick to go through even during a heavily +%% loaded system. gen_tcp does not have a +%% non-blocking send operation exposed in its API +%% and we don't want to run the distribution +%% controller under high priority. Therefore this +%% sparate process with max prio that dispatches +%% ticks. +%% +dist_cntrlr_tick_handler(Socket) -> + receive + tick -> + %% May block due to busy port... + sock_send(Socket, ""); + _ -> + ok + end, + dist_cntrlr_tick_handler(Socket). + +spawn_dist_cntrlr(Socket) -> + spawn_opt(?MODULE, dist_cntrlr_setup, [Socket], + [{priority, max}] ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS). + +dist_cntrlr_setup(Socket) -> + TickHandler = spawn_opt(?MODULE, dist_cntrlr_tick_handler, + [Socket], + [link, {priority, max}] + ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS), + dist_cntrlr_setup_loop(Socket, TickHandler, undefined). + +%% +%% During the handshake phase we loop in dist_cntrlr_setup(). +%% When the connection is up we spawn an input handler and +%% continue as output handler. +%% +dist_cntrlr_setup_loop(Socket, TickHandler, Sup) -> + receive + {tcp_closed, Socket} -> + exit(connection_closed); + + {Ref, From, {supervisor, Pid}} -> + Res = link(Pid), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Pid); + + {Ref, From, tick_handler} -> + From ! {Ref, TickHandler}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, socket} -> + From ! {Ref, Socket}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {send, Packet}} -> + Res = gen_tcp:send(Socket, Packet), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {recv, Length, Timeout}} -> + Res = gen_tcp:recv(Socket, Length, Timeout), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, getll} -> + From ! {Ref, {ok, self()}}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {address, Node}} -> + Res = case inet:peername(Socket) of + {ok, Address} -> + case split_node(atom_to_list(Node), $@, []) of + [_,Host] -> + #net_address{address=Address,host=Host, + protocol=tcp, family=inet}; + _ -> + {error, no_node} + end + end, + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, pre_nodeup} -> + Res = inet:setopts(Socket, + [{active, false}, + {packet, 4}, + nodelay()]), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, post_nodeup} -> + Res = inet:setopts(Socket, + [{active, false}, + {packet, 4}, + nodelay()]), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {handshake_complete, _Node, DHandle}} -> + From ! {Ref, ok}, + %% Handshake complete! Begin dispatching traffic... + + %% We use separate process for dispatching input. This + %% is not necessary, but it enables parallel execution + %% of independent work loads at the same time as it + %% simplifies the the implementation... + InputHandler = spawn_opt(?MODULE, dist_cntrlr_input_setup, + [DHandle, Socket, Sup], + [link] ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS), + + flush_controller(InputHandler, Socket), + gen_tcp:controlling_process(Socket, InputHandler), + flush_controller(InputHandler, Socket), + + ok = erlang:dist_ctrl_input_handler(DHandle, InputHandler), + + InputHandler ! DHandle, + + %% From now on we execute on normal priority + process_flag(priority, normal), + erlang:dist_ctrl_get_data_notification(DHandle), + dist_cntrlr_output_loop(DHandle, Socket) + end. + +%% We use active 10 for good throughput while still +%% maintaining back-pressure if the input controller +%% isn't able to handle all incoming messages... +-define(ACTIVE_INPUT, 10). + +dist_cntrlr_input_setup(DHandle, Socket, Sup) -> + link(Sup), + %% Ensure we don't try to put data before registerd + %% as input handler... + receive + DHandle -> + dist_cntrlr_input_loop(DHandle, Socket, 0) + end. + +dist_cntrlr_input_loop(DHandle, Socket, N) when N =< ?ACTIVE_INPUT/2 -> + inet:setopts(Socket, [{active, ?ACTIVE_INPUT - N}]), + dist_cntrlr_input_loop(DHandle, Socket, ?ACTIVE_INPUT); +dist_cntrlr_input_loop(DHandle, Socket, N) -> + receive + {tcp_closed, Socket} -> + %% Connection to remote node terminated... + exit(connection_closed); + + {tcp, Socket, Data} -> + %% Incoming data from remote node... + try erlang:dist_ctrl_put_data(DHandle, Data) + catch _ : _ -> death_row() + end, + dist_cntrlr_input_loop(DHandle, Socket, N-1); + + _ -> + %% Ignore... + dist_cntrlr_input_loop(DHandle, Socket, N) + end. + +dist_cntrlr_send_data(DHandle, Socket) -> + case erlang:dist_ctrl_get_data(DHandle) of + none -> + erlang:dist_ctrl_get_data_notification(DHandle); + Data -> + sock_send(Socket, Data), + dist_cntrlr_send_data(DHandle, Socket) + end. + + +dist_cntrlr_output_loop(DHandle, Socket) -> + receive + dist_data -> + %% Outgoing data from this node... + try dist_cntrlr_send_data(DHandle, Socket) + catch _ : _ -> death_row() + end, + dist_cntrlr_output_loop(DHandle, Socket); + + {send, From, Ref, Data} -> + %% This is for testing only! + %% + %% Needed by some OTP distribution + %% test suites... + sock_send(Socket, Data), + From ! {Ref, ok}, + dist_cntrlr_output_loop(DHandle, Socket); + + _ -> + %% Drop garbage message... + dist_cntrlr_output_loop(DHandle, Socket) + + end. + +sock_send(Socket, Data) -> + try gen_tcp:send(Socket, Data) of + ok -> ok; + {error, Reason} -> death_row({send_error, Reason}) + catch + Type : Reason -> death_row({send_error, {Type, Reason}}) + end. + +death_row() -> + death_row(connection_closed). + +death_row(normal) -> + %% We do not want to exit with normal + %% exit reason since it wont bring down + %% linked processes... + death_row(); +death_row(Reason) -> + %% When the connection is on its way down operations + %% begin to fail. We catch the failures and call + %% this function waiting for termination. We should + %% be terminated by one of our links to the other + %% involved parties that began bringing the + %% connection down. By waiting for termination we + %% avoid altering the exit reason for the connection + %% teardown. We however limit the wait to 5 seconds + %% and bring down the connection ourselves if not + %% terminated... + receive after 5000 -> exit(Reason) end. diff --git a/lib/kernel/include/dist.hrl b/lib/kernel/include/dist.hrl index d6bccdf474..db4a5eaebc 100644 --- a/lib/kernel/include/dist.hrl +++ b/lib/kernel/include/dist.hrl @@ -40,3 +40,33 @@ -define(DFLAG_UTF8_ATOMS, 16#10000). -define(DFLAG_MAP_TAG, 16#20000). -define(DFLAG_BIG_CREATION, 16#40000). +-define(DFLAG_SEND_SENDER, 16#80000). + +%% DFLAGs that require strict ordering or:ed together... +-define(DFLAGS_STRICT_ORDER_DELIVERY, + ?DFLAG_DIST_HDR_ATOM_CACHE). + + +%% Also update dflag2str() in ../src/dist_util.erl +%% when adding flags... + +-define(DFLAGS_ALL, + (?DFLAG_PUBLISHED + bor ?DFLAG_ATOM_CACHE + bor ?DFLAG_EXTENDED_REFERENCES + bor ?DFLAG_DIST_MONITOR + bor ?DFLAG_FUN_TAGS + bor ?DFLAG_DIST_MONITOR_NAME + bor ?DFLAG_HIDDEN_ATOM_CACHE + bor ?DFLAG_NEW_FUN_TAGS + bor ?DFLAG_EXTENDED_PIDS_PORTS + bor ?DFLAG_EXPORT_PTR_TAG + bor ?DFLAG_BIT_BINARIES + bor ?DFLAG_NEW_FLOATS + bor ?DFLAG_UNICODE_IO + bor ?DFLAG_DIST_HDR_ATOM_CACHE + bor ?DFLAG_SMALL_ATOM_TAGS + bor ?DFLAG_UTF8_ATOMS + bor ?DFLAG_MAP_TAG + bor ?DFLAG_BIG_CREATION + bor ?DFLAG_SEND_SENDER)). diff --git a/lib/kernel/include/dist_util.hrl b/lib/kernel/include/dist_util.hrl index e3d2fe0eb6..eeb0f8dd43 100644 --- a/lib/kernel/include/dist_util.hrl +++ b/lib/kernel/include/dist_util.hrl @@ -29,9 +29,9 @@ -endif. -ifdef(dist_trace). --define(trace(Fmt,Args), io:format("~p ~p:~s",[erlang:timestamp(),node(),lists:flatten(io_lib:format(Fmt, Args))])). +-define(trace(Fmt,Args), io:format("~p ~p:~s",[erlang:convert_time_unit(erlang:monotonic_time()-erlang:system_info(start_time), native, microsecond),node(),lists:flatten(io_lib:format(Fmt, Args))])). % Use the one below for config-file (early boot) connection tracing -%-define(trace(Fmt,Args), erlang:display([erlang:now(),node(),lists:flatten(io_lib:format(Fmt, Args))])). +%-define(trace(Fmt,Args), erlang:display([erlang:convert_time_unit(erlang:monotonic_time()-erlang:system_info(start_time), native, microsecond),node(),lists:flatten(io_lib:format(Fmt, Args))])). -define(trace_factor,8). -else. -define(trace(Fmt,Args), ok). @@ -78,7 +78,13 @@ %% New in kernel-5.1 (OTP 19.1): mf_setopts, %% netkernel:setopts on active connection - mf_getopts %% netkernel:getopts on active connection + mf_getopts, %% netkernel:getopts on active connection + + %% New in kernel-6.0 (OTP 21.0) + f_handshake_complete, %% Notify handshake complete + add_flags, %% dflags to add + reject_flags, %% dflags not to use (not all can be rejected) + require_flags %% dflags that are required }). diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl index b3507e5d13..08bd5946cd 100644 --- a/lib/kernel/src/dist_util.erl +++ b/lib/kernel/src/dist_util.erl @@ -74,6 +74,48 @@ ticked = 0 }). +dflag2str(?DFLAG_PUBLISHED) -> + "PUBLISHED"; +dflag2str(?DFLAG_ATOM_CACHE) -> + "ATOM_CACHE"; +dflag2str(?DFLAG_EXTENDED_REFERENCES) -> + "EXTENDED_REFERENCES"; +dflag2str(?DFLAG_DIST_MONITOR) -> + "DIST_MONITOR"; +dflag2str(?DFLAG_FUN_TAGS) -> + "FUN_TAGS"; +dflag2str(?DFLAG_DIST_MONITOR_NAME) -> + "DIST_MONITOR_NAME"; +dflag2str(?DFLAG_HIDDEN_ATOM_CACHE) -> + "HIDDEN_ATOM_CACHE"; +dflag2str(?DFLAG_NEW_FUN_TAGS) -> + "NEW_FUN_TAGS"; +dflag2str(?DFLAG_EXTENDED_PIDS_PORTS) -> + "EXTENDED_PIDS_PORTS"; +dflag2str(?DFLAG_EXPORT_PTR_TAG) -> + "EXPORT_PTR_TAG"; +dflag2str(?DFLAG_BIT_BINARIES) -> + "BIT_BINARIES"; +dflag2str(?DFLAG_NEW_FLOATS) -> + "NEW_FLOATS"; +dflag2str(?DFLAG_UNICODE_IO) -> + "UNICODE_IO"; +dflag2str(?DFLAG_DIST_HDR_ATOM_CACHE) -> + "DIST_HDR_ATOM_CACHE"; +dflag2str(?DFLAG_SMALL_ATOM_TAGS) -> + "SMALL_ATOM_TAGS"; +dflag2str(?DFLAG_UTF8_ATOMS) -> + "UTF8_ATOMS"; +dflag2str(?DFLAG_MAP_TAG) -> + "MAP_TAG"; +dflag2str(?DFLAG_BIG_CREATION) -> + "BIG_CREATION"; +dflag2str(?DFLAG_SEND_SENDER) -> + "SEND_SENDER"; +dflag2str(_) -> + "UNKNOWN". + + remove_flag(Flag, Flags) -> case Flags band Flag of 0 -> @@ -82,13 +124,13 @@ remove_flag(Flag, Flags) -> Flags - Flag end. -adjust_flags(ThisFlags, OtherFlags) -> +adjust_flags(ThisFlags, OtherFlags, RejectFlags) -> case (?DFLAG_PUBLISHED band ThisFlags) band OtherFlags of 0 -> {remove_flag(?DFLAG_PUBLISHED, ThisFlags), remove_flag(?DFLAG_PUBLISHED, OtherFlags)}; _ -> - {ThisFlags, OtherFlags} + {ThisFlags, OtherFlags band (bnot RejectFlags)} end. publish_flag(hidden, _) -> @@ -101,36 +143,71 @@ publish_flag(_, OtherNode) -> 0 end. -make_this_flags(RequestType, OtherNode) -> - publish_flag(RequestType, OtherNode) bor - %% The parenthesis below makes the compiler generate better code. - (?DFLAG_EXPORT_PTR_TAG bor - ?DFLAG_EXTENDED_PIDS_PORTS bor - ?DFLAG_EXTENDED_REFERENCES bor - ?DFLAG_DIST_MONITOR bor - ?DFLAG_FUN_TAGS bor - ?DFLAG_DIST_MONITOR_NAME bor - ?DFLAG_HIDDEN_ATOM_CACHE bor - ?DFLAG_NEW_FUN_TAGS bor - ?DFLAG_BIT_BINARIES bor - ?DFLAG_NEW_FLOATS bor - ?DFLAG_UNICODE_IO bor - ?DFLAG_DIST_HDR_ATOM_CACHE bor - ?DFLAG_SMALL_ATOM_TAGS bor - ?DFLAG_UTF8_ATOMS bor - ?DFLAG_MAP_TAG bor - ?DFLAG_BIG_CREATION). - -handshake_other_started(#hs_data{request_type=ReqType}=HSData0) -> +-define(DFLAGS_REMOVABLE, + (?DFLAG_DIST_HDR_ATOM_CACHE + bor ?DFLAG_HIDDEN_ATOM_CACHE + bor ?DFLAG_ATOM_CACHE)). + +-define(DFLAGS_ADDABLE, + (?DFLAGS_ALL + band (bnot (?DFLAG_PUBLISHED + bor ?DFLAG_HIDDEN_ATOM_CACHE + bor ?DFLAG_ATOM_CACHE)))). + +-define(DFLAGS_THIS_DEFAULT, + (?DFLAG_EXPORT_PTR_TAG + bor ?DFLAG_EXTENDED_PIDS_PORTS + bor ?DFLAG_EXTENDED_REFERENCES + bor ?DFLAG_DIST_MONITOR + bor ?DFLAG_FUN_TAGS + bor ?DFLAG_DIST_MONITOR_NAME + bor ?DFLAG_NEW_FUN_TAGS + bor ?DFLAG_BIT_BINARIES + bor ?DFLAG_NEW_FLOATS + bor ?DFLAG_UNICODE_IO + bor ?DFLAG_DIST_HDR_ATOM_CACHE + bor ?DFLAG_SMALL_ATOM_TAGS + bor ?DFLAG_UTF8_ATOMS + bor ?DFLAG_MAP_TAG + bor ?DFLAG_BIG_CREATION + bor ?DFLAG_SEND_SENDER)). + +make_this_flags(RequestType, AddFlags, RemoveFlags, OtherNode) -> + case RemoveFlags band (bnot ?DFLAGS_REMOVABLE) of + 0 -> ok; + Rerror -> exit({"Rejecting non rejectable flags", Rerror}) + end, + case AddFlags band (bnot ?DFLAGS_ADDABLE) of + 0 -> ok; + Aerror -> exit({"Adding non addable flags", Aerror}) + end, + Flgs0 = ?DFLAGS_THIS_DEFAULT, + Flgs1 = Flgs0 bor publish_flag(RequestType, OtherNode), + Flgs2 = Flgs1 bor AddFlags, + Flgs3 = Flgs2 band (bnot (?DFLAG_HIDDEN_ATOM_CACHE + bor ?DFLAG_ATOM_CACHE)), + Flgs3 band (bnot RemoveFlags). + +handshake_other_started(#hs_data{request_type=ReqType, + add_flags=AddFlgs0, + reject_flags=RejFlgs0, + require_flags=ReqFlgs0}=HSData0) -> + AddFlgs = convert_flags(AddFlgs0), + RejFlgs = convert_flags(RejFlgs0), + ReqFlgs = convert_flags(ReqFlgs0), {PreOtherFlags,Node,Version} = recv_name(HSData0), - PreThisFlags = make_this_flags(ReqType, Node), + PreThisFlags = make_this_flags(ReqType, AddFlgs, RejFlgs, Node), {ThisFlags, OtherFlags} = adjust_flags(PreThisFlags, - PreOtherFlags), + PreOtherFlags, + RejFlgs), HSData = HSData0#hs_data{this_flags=ThisFlags, other_flags=OtherFlags, other_version=Version, other_node=Node, - other_started=true}, + other_started=true, + add_flags=AddFlgs, + reject_flags=RejFlgs, + require_flags=ReqFlgs}, check_dflags(HSData), is_allowed(HSData), ?debug({"MD5 connection from ~p (V~p)~n", @@ -165,23 +242,18 @@ is_allowed(#hs_data{other_node = Node, end. %% -%% Check that both nodes can handle the same types of extended -%% node containers. If they can not, abort the connection. +%% Check mandatory flags... %% check_dflags(#hs_data{other_node = Node, other_flags = OtherFlags, - other_started = OtherStarted} = HSData) -> - - Mandatory = [{?DFLAG_EXTENDED_REFERENCES, "EXTENDED_REFERENCES"}, - {?DFLAG_EXTENDED_PIDS_PORTS, "EXTENDED_PIDS_PORTS"}, - {?DFLAG_UTF8_ATOMS, "UTF8_ATOMS"}], - Missing = lists:filtermap(fun({Bit, Str}) -> - case Bit band OtherFlags of - Bit -> false; - 0 -> {true, Str} - end - end, - Mandatory), + other_started = OtherStarted, + require_flags = RequiredFlags} = HSData) -> + Mandatory = ((?DFLAG_EXTENDED_REFERENCES + bor ?DFLAG_EXTENDED_PIDS_PORTS + bor ?DFLAG_UTF8_ATOMS) + bor RequiredFlags), + Missing = check_mandatory(0, ?DFLAGS_ALL, Mandatory, + OtherFlags, []), case Missing of [] -> ok; @@ -201,6 +273,22 @@ check_dflags(#hs_data{other_node = Node, ?shutdown2(Node, {check_dflags_failed, Missing}) end. +check_mandatory(_Bit, 0, _Mandatory, _OtherFlags, Missing) -> + Missing; +check_mandatory(Bit, Left, Mandatory, OtherFlags, Missing) -> + DFlag = (1 bsl Bit), + NewLeft = Left band (bnot DFlag), + NewMissing = case {DFlag band Mandatory, + DFlag band OtherFlags} of + {DFlag, 0} -> + %% Mandatory and missing... + [dflag2str(DFlag) | Missing]; + _ -> + %% Not mandatory or present... + Missing + end, + check_mandatory(Bit+1, NewLeft, Mandatory, OtherFlags, NewMissing). + %% No nodedown will be sent if we fail before this process has %% succeeded to mark the node as pending. @@ -314,13 +402,24 @@ flush_down() -> end. handshake_we_started(#hs_data{request_type=ReqType, - other_node=Node}=PreHSData) -> - PreThisFlags = make_this_flags(ReqType, Node), - HSData = PreHSData#hs_data{this_flags=PreThisFlags}, + other_node=Node, + add_flags=AddFlgs0, + reject_flags=RejFlgs0, + require_flags=ReqFlgs0}=PreHSData) -> + AddFlgs = convert_flags(AddFlgs0), + RejFlgs = convert_flags(RejFlgs0), + ReqFlgs = convert_flags(ReqFlgs0), + PreThisFlags = make_this_flags(ReqType, AddFlgs, RejFlgs, Node), + HSData = PreHSData#hs_data{this_flags = PreThisFlags, + add_flags = AddFlgs, + reject_flags = RejFlgs, + require_flags = ReqFlgs}, send_name(HSData), recv_status(HSData), {PreOtherFlags,ChallengeA} = recv_challenge(HSData), - {ThisFlags,OtherFlags} = adjust_flags(PreThisFlags, PreOtherFlags), + {ThisFlags,OtherFlags} = adjust_flags(PreThisFlags, + PreOtherFlags, + RejFlgs), NewHSData = HSData#hs_data{this_flags = ThisFlags, other_flags = OtherFlags, other_started = false}, @@ -336,15 +435,16 @@ handshake_we_started(#hs_data{request_type=ReqType, handshake_we_started(OldHsData) when element(1,OldHsData) =:= hs_data -> handshake_we_started(convert_old_hsdata(OldHsData)). -convert_old_hsdata({hs_data, KP, ON, TN, S, T, TF, A, OV, OF, OS, FS, FR, - FS_PRE, FS_POST, FG, FA, MFT, MFG, RT}) -> - #hs_data{ - kernel_pid = KP, other_node = ON, this_node = TN, socket = S, timer = T, - this_flags = TF, allowed = A, other_version = OV, other_flags = OF, - other_started = OS, f_send = FS, f_recv = FR, f_setopts_pre_nodeup = FS_PRE, - f_setopts_post_nodeup = FS_POST, f_getll = FG, f_address = FA, - mf_tick = MFT, mf_getstat = MFG, request_type = RT}. +convert_old_hsdata(OldHsData) -> + OHSDL = tuple_to_list(OldHsData), + NoMissing = tuple_size(#hs_data{}) - tuple_size(OldHsData), + true = NoMissing > 0, + list_to_tuple(OHSDL ++ lists:duplicate(NoMissing, undefined)). +convert_flags(Flags) when is_integer(Flags) -> + Flags; +convert_flags(_Undefined) -> + 0. %% -------------------------------------------------------------- %% The connection has been established. @@ -359,15 +459,20 @@ connection(#hs_data{other_node = Node, PType = publish_type(HSData#hs_data.other_flags), case FPreNodeup(Socket) of ok -> - do_setnode(HSData), % Succeeds or exits the process. + DHandle = do_setnode(HSData), % Succeeds or exits the process. Address = FAddress(Socket,Node), mark_nodeup(HSData,Address), case FPostNodeup(Socket) of ok -> + case HSData#hs_data.f_handshake_complete of + undefined -> ok; + HsComplete -> HsComplete(Socket, Node, DHandle) + end, con_loop({HSData#hs_data.kernel_pid, Node, Socket, PType, + DHandle, HSData#hs_data.mf_tick, HSData#hs_data.mf_getstat, HSData#hs_data.mf_setopts, @@ -425,18 +530,16 @@ do_setnode(#hs_data{other_node = Node, socket = Socket, [Node, Port, {publish_type(Flags), '(', Flags, ')', Version}]), - case (catch - erlang:setnode(Node, Port, - {Flags, Version, '', ''})) of - {'EXIT', {system_limit, _}} -> + try + erlang:setnode(Node, Port, {Flags, Version, '', ''}) + catch + error:system_limit -> error_msg("** Distribution system limit reached, " "no table space left for node ~w ** ~n", [Node]), ?shutdown(Node); - {'EXIT', Other} -> - exit(Other); - _Else -> - ok + error:Other -> + exit({Other, erlang:get_stacktrace()}) end; _ -> error_msg("** Distribution connection error, " @@ -468,7 +571,13 @@ mark_nodeup(#hs_data{kernel_pid = Kernel, ?shutdown(Node) end. -con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=ConData, +getstat(DHandle, _Socket, undefined) -> + erlang:dist_get_stat(DHandle); +getstat(_DHandle, Socket, MFGetstat) -> + MFGetstat(Socket). + +con_loop({Kernel, Node, Socket, Type, DHandle, MFTick, MFGetstat, + MFSetOpts, MFGetOpts}=ConData, Tick) -> receive {tcp_closed, Socket} -> @@ -476,7 +585,7 @@ con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=C {Kernel, disconnect} -> ?shutdown2(Node, disconnected); {Kernel, aux_tick} -> - case MFGetstat(Socket) of + case getstat(DHandle, Socket, MFGetstat) of {ok, _, _, PendWrite} -> send_tick(Socket, PendWrite, MFTick); _ -> @@ -484,7 +593,7 @@ con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=C end, con_loop(ConData, Tick); {Kernel, tick} -> - case send_tick(Socket, Tick, Type, + case send_tick(DHandle, Socket, Tick, Type, MFTick, MFGetstat) of {ok, NewTick} -> con_loop(ConData, NewTick); @@ -497,7 +606,7 @@ con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=C ?shutdown2(Node, send_net_tick_failed) end; {From, get_status} -> - case MFGetstat(Socket) of + case getstat(DHandle, Socket, MFGetstat) of {ok, Read, Write, _} -> From ! {self(), get_status, {ok, Read, Write}}, con_loop(ConData, Tick); @@ -735,14 +844,14 @@ send_status(#hs_data{socket = Socket, other_node = Node, %% we haven't read anything as a hidden node only ticks when it receives %% a TICK !! -send_tick(Socket, Tick, Type, MFTick, MFGetstat) -> +send_tick(DHandle, Socket, Tick, Type, MFTick, MFGetstat) -> #tick{tick = T0, read = Read, write = Write, ticked = Ticked} = Tick, T = T0 + 1, T1 = T rem 4, - case MFGetstat(Socket) of + case getstat(DHandle, Socket, MFGetstat) of {ok, Read, _, _} when Ticked =:= T -> {error, not_responding}; {ok, Read, W, Pend} when Type =:= hidden -> @@ -771,11 +880,10 @@ send_tick(Socket, Tick, Type, MFTick, MFGetstat) -> Error end. -send_tick(Socket, 0, MFTick) -> - MFTick(Socket); -send_tick(_, _Pend, _) -> - %% Dont send tick if pending write. - ok. +send_tick(_, Pend, _) when Pend /= false, Pend /= 0 -> + ok; %% Dont send tick if pending write. +send_tick(Socket, _Pend, MFTick) -> + MFTick(Socket). %% ------------------------------------------------------------ %% Connection setup timeout timer. diff --git a/lib/kernel/src/erl_epmd.erl b/lib/kernel/src/erl_epmd.erl index 7bc9e2ede3..f96bc88913 100644 --- a/lib/kernel/src/erl_epmd.erl +++ b/lib/kernel/src/erl_epmd.erl @@ -79,7 +79,13 @@ port_please(Node, EpmdAddr, Timeout) -> port_please1(Node,HostName, Timeout) -> - case inet:gethostbyname(HostName, inet, Timeout) of + Family = case inet_db:res_option(inet6) of + true -> + inet6; + false -> + inet + end, + case inet:gethostbyname(HostName, Family, Timeout) of {ok,{hostent, _Name, _ , _Af, _Size, [EpmdAddr | _]}} -> get_port(Node, EpmdAddr, Timeout); Else -> diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index ad92aafc2f..480db6814e 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -33,10 +33,10 @@ -export([breakpoint/2, disassemble/1, display/1, dist_ext_to_term/2, dump_monitors/1, dump_links/1, flat_size/1, - get_internal_state/1, instructions/0, lock_counters/1, + get_internal_state/1, instructions/0, map_info/1, same/2, set_internal_state/2, - size_shared/1, copy_shared/1, dirty_cpu/2, dirty_io/2, - dirty/3]). + size_shared/1, copy_shared/1, dirty_cpu/2, dirty_io/2, dirty/3, + lcnt_control/1, lcnt_control/2, lcnt_collect/0, lcnt_clear/0]). -spec breakpoint(MFA, Flag) -> non_neg_integer() when MFA :: {Module :: module(), @@ -142,12 +142,31 @@ ic(F) when is_function(F) -> io:format("Total: ~w~n",[lists:sum([C||{_I,C}<-Is])]), R. --spec lock_counters(info) -> term(); - (clear) -> ok; - ({copy_save, boolean()}) -> boolean(); - ({process_locks, boolean()}) -> boolean(). +-spec lcnt_control + (copy_save, boolean()) -> ok; + (mask, list(atom())) -> ok. -lock_counters(_) -> +lcnt_control(_Option, _Value) -> + erlang:nif_error(undef). + +-spec lcnt_control + (copy_save) -> boolean(); + (mask) -> list(atom()). + +lcnt_control(_Option) -> + erlang:nif_error(undef). + +-type lcnt_lock_info() :: {atom(), term(), atom(), term()}. + +-spec lcnt_collect() -> + list({duration, {non_neg_integer(), non_neg_integer()}} | + {locks, list(lcnt_lock_info())}). + +lcnt_collect() -> + erlang:nif_error(undef). + +-spec lcnt_clear() -> ok. +lcnt_clear() -> erlang:nif_error(undef). -spec same(Term1, Term2) -> boolean() when diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index 6aef5476f1..dc20c21c77 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -151,7 +151,8 @@ %%% --------------------------------- --spec get_rc() -> [{Par :: any(), Val :: any()}]. +-spec get_rc() -> [{Par :: atom(), Val :: any()} | + {Par :: atom(), Val1 :: any(), Val2 :: any()}]. get_rc() -> inet_db:get_rc(). diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index e150938487..2a11b04310 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -120,6 +120,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-9.0", "stdlib-3.0", "sasl-3.0"]} + {runtime_dependencies, ["erts-9.1", "stdlib-3.0", "sasl-3.0"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 77085b2064..fc5417597f 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -18,7 +18,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"5\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* + [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.* %% Down to - max one major revision back - [{<<"5\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* + [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* }. diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index ddda396713..fb4faea420 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -423,8 +423,8 @@ handle_call({connect, Type, Node}, From, State) -> {ok, SetupPid} -> Owners = [{SetupPid, Node} | State#state.conn_owners], {noreply,State#state{conn_owners=Owners}}; - _ -> - ?connect_failure(Node, {setup_call, failed}), + _Error -> + ?connect_failure(Node, {setup_call, failed, _Error}), async_reply({reply, false, State}, From) end end; diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index b794d4f45e..99ea4210bd 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -175,6 +175,18 @@ server_loop(Iport, Oport, Curr, User, Gr, {Resp, IOQ} = IOQueue) -> {Iport,eof} -> Curr ! {self(),eof}, server_loop(Iport, Oport, Curr, User, Gr, IOQueue); + + %% We always handle geometry and unicode requests + {Requester,tty_geometry} -> + Requester ! {self(),tty_geometry,get_tty_geometry(Iport)}, + server_loop(Iport, Oport, Curr, User, Gr, IOQueue); + {Requester,get_unicode_state} -> + Requester ! {self(),get_unicode_state,get_unicode_state(Iport)}, + server_loop(Iport, Oport, Curr, User, Gr, IOQueue); + {Requester,set_unicode_state, Bool} -> + Requester ! {self(),set_unicode_state,set_unicode_state(Iport,Bool)}, + server_loop(Iport, Oport, Curr, User, Gr, IOQueue); + Req when element(1,Req) =:= User orelse element(1,Req) =:= Curr, tuple_size(Req) =:= 2 orelse tuple_size(Req) =:= 3 -> %% We match {User|Curr,_}|{User|Curr,_,_} @@ -224,21 +236,16 @@ server_loop(Iport, Oport, Curr, User, Gr, {Resp, IOQ} = IOQueue) -> _ -> % not current, just remove it server_loop(Iport, Oport, Curr, User, gr_del_pid(Gr, Pid), IOQueue) end; + {Requester, {put_chars_sync, _, _, Reply}} -> + %% We need to ack the Req otherwise originating process will hang forever + %% Do discard the output to non visible shells (as was done previously) + Requester ! {reply, Reply}, + server_loop(Iport, Oport, Curr, User, Gr, IOQueue); _X -> - %% Ignore unknown messages. - server_loop(Iport, Oport, Curr, User, Gr, IOQueue) + %% Ignore unknown messages. + server_loop(Iport, Oport, Curr, User, Gr, IOQueue) end. -%% We always handle geometry and unicode requests -handle_req({Curr,tty_geometry},Iport,_Oport,IOQueue) -> - Curr ! {self(),tty_geometry,get_tty_geometry(Iport)}, - IOQueue; -handle_req({Curr,get_unicode_state},Iport,_Oport,IOQueue) -> - Curr ! {self(),get_unicode_state,get_unicode_state(Iport)}, - IOQueue; -handle_req({Curr,set_unicode_state, Bool},Iport,_Oport,IOQueue) -> - Curr ! {self(),set_unicode_state,set_unicode_state(Iport,Bool)}, - IOQueue; handle_req(next,Iport,Oport,{false,IOQ}=IOQueue) -> case queue:out(IOQ) of {empty,_} -> diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 929f66d400..331864b5de 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -50,9 +50,8 @@ killing_acceptor/1,killing_multi_acceptors/1,killing_multi_acceptors2/1, several_accepts_in_one_go/1, accept_system_limit/1, active_once_closed/1, send_timeout/1, send_timeout_active/1, - otp_7731/1, zombie_sockets/1, otp_7816/1, otp_8102/1, - wrapping_oct/0, wrapping_oct/1, - otp_9389/1]). + otp_7731/1, zombie_sockets/1, otp_7816/1, otp_8102/1, + wrapping_oct/0, wrapping_oct/1, otp_9389/1, otp_13939/1]). %% Internal exports. -export([sender/3, not_owner/1, passive_sockets_server/2, priority_server/1, @@ -3014,3 +3013,42 @@ ok({ok,V}) -> V. get_hostname(Name) -> "@"++Host = lists:dropwhile(fun(C) -> C =/= $@ end, atom_to_list(Name)), Host. + +otp_13939(doc) -> + ["Check that writing to a remotely closed socket doesn't block forever " + "when exit_on_close is false."]; +otp_13939(suite) -> + []; +otp_13939(Config) when is_list(Config) -> + {Pid, Ref} = spawn_opt( + fun() -> + {ok, Listener} = gen_tcp:listen(0, [{exit_on_close, false}]), + {ok, Port} = inet:port(Listener), + + spawn_link( + fun() -> + {ok, Client} = gen_tcp:connect("localhost", Port, + [{active, false}]), + ok = gen_tcp:close(Client) + end), + + {ok, Accepted} = gen_tcp:accept(Listener), + + ok = gen_tcp:send(Accepted, <<0:(10*1024*1024*8)>>), + + %% The bug surfaces when there's a delay between the send + %% operations; inet:getstat is a red herring. + timer:sleep(100), + + {error, Code} = gen_tcp:send(Accepted, <<0:(10*1024*1024*8)>>), + ct:pal("gen_tcp:send returned ~p~n", [Code]) + end, [link, monitor]), + + receive + {'DOWN', Ref, process, Pid, normal} -> + ok + after 1000 -> + demonitor(Ref, [flush]), + exit(Pid, normal), + ct:fail("Server process blocked on send.") + end. diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl index aa616d43d6..8364fe7cdc 100644 --- a/lib/kernel/test/gen_udp_SUITE.erl +++ b/lib/kernel/test/gen_udp_SUITE.erl @@ -295,21 +295,9 @@ bad_address(Config) when is_list(Config) -> %% are received per in/out scheduling, which should be %% the same as the read_packets parameter. %% -%% What happens on the SMP emulator remains to be seen... -%% %% OTP-6249 UDP option for number of packet reads. read_packets(Config) when is_list(Config) -> - case erlang:system_info(smp_support) of - false -> - read_packets_1(); - true -> - %% We would need some new sort of tracing to test this - %% option reliably in an SMP emulator. - {skip,"SMP emulator"} - end. - -read_packets_1() -> N1 = 5, N2 = 7, {ok,R} = gen_udp:open(0, [{read_packets,N1}]), diff --git a/lib/kernel/test/zlib_SUITE.erl b/lib/kernel/test/zlib_SUITE.erl index 4b67fce9a8..a44ef106d3 100644 --- a/lib/kernel/test/zlib_SUITE.erl +++ b/lib/kernel/test/zlib_SUITE.erl @@ -885,29 +885,23 @@ split_bin(Last,Acc) -> %% Check concurrent access to zlib driver. smp(Config) -> - case erlang:system_info(smp_support) of - true -> - NumOfProcs = lists:min([8,erlang:system_info(schedulers)]), - io:format("smp starting ~p workers\n",[NumOfProcs]), - - %% Tests to run in parallel. - Funcs = [zip_usage, gz_usage, compress_usage, dictionary_usage, - crc, adler], - - %% We get all function arguments here to avoid repeated parallel - %% file read access. - FnAList = lists:map(fun(F) -> {F,?MODULE:F({get_arg,Config})} - end, Funcs), - - Pids = [spawn_link(?MODULE, worker, [rand:uniform(9999), - list_to_tuple(FnAList), - self()]) - || _ <- lists:seq(1,NumOfProcs)], - wait_pids(Pids); - - false -> - {skipped,"No smp support"} - end. + NumOfProcs = lists:min([8,erlang:system_info(schedulers)]), + io:format("smp starting ~p workers\n",[NumOfProcs]), + + %% Tests to run in parallel. + Funcs = [zip_usage, gz_usage, compress_usage, dictionary_usage, + crc, adler], + + %% We get all function arguments here to avoid repeated parallel + %% file read access. + FnAList = lists:map(fun(F) -> {F,?MODULE:F({get_arg,Config})} + end, Funcs), + + Pids = [spawn_link(?MODULE, worker, [rand:uniform(9999), + list_to_tuple(FnAList), + self()]) + || _ <- lists:seq(1,NumOfProcs)], + wait_pids(Pids). worker(Seed, FnATpl, Parent) -> diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 4edecd8969..c9463241d1 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 5.3 +KERNEL_VSN = 5.3.1 diff --git a/lib/observer/src/observer_lib.erl b/lib/observer/src/observer_lib.erl index c7ee294719..8b6036f52a 100644 --- a/lib/observer/src/observer_lib.erl +++ b/lib/observer/src/observer_lib.erl @@ -297,6 +297,8 @@ to_str(No) when is_integer(No) -> integer_to_list(No); to_str(Float) when is_float(Float) -> io_lib:format("~.3f", [Float]); +to_str({trunc, Float}) when is_float(Float) -> + float_to_list(Float, [{decimals,0}]); to_str(Term) -> io_lib:format("~w", [Term]). diff --git a/lib/observer/src/observer_sys_wx.erl b/lib/observer/src/observer_sys_wx.erl index db86c05bed..2e1af3ada9 100644 --- a/lib/observer/src/observer_sys_wx.erl +++ b/lib/observer/src/observer_sys_wx.erl @@ -48,7 +48,7 @@ start_link(Notebook, Parent, Config) -> init([Notebook, Parent, Config]) -> SysInfo = observer_backend:sys_info(), - {Sys, Mem, Cpu, Stats} = info_fields(), + {Sys, Mem, Cpu, Stats, Limits} = info_fields(), Panel = wxPanel:new(Notebook), Sizer = wxBoxSizer:new(?wxVERTICAL), HSizer0 = wxBoxSizer:new(?wxHORIZONTAL), @@ -63,17 +63,26 @@ init([Notebook, Parent, Config]) -> wxSizer:add(HSizer1, FPanel2, [{flag, ?wxEXPAND}, {proportion, 1}]), wxSizer:add(HSizer1, FPanel3, [{flag, ?wxEXPAND}, {proportion, 1}]), + HSizer2 = wxBoxSizer:new(?wxHORIZONTAL), + {FPanel4, _FSizer4, Fields4} = observer_lib:display_info(Panel, observer_lib:fill_info(Limits, SysInfo)), + wxSizer:add(HSizer2, FPanel4, [{flag, ?wxEXPAND}, {proportion, 1}]), + + BorderFlags = ?wxLEFT bor ?wxRIGHT, wxSizer:add(Sizer, HSizer0, [{flag, ?wxEXPAND bor BorderFlags bor ?wxTOP}, {proportion, 0}, {border, 5}]), wxSizer:add(Sizer, HSizer1, [{flag, ?wxEXPAND bor BorderFlags bor ?wxBOTTOM}, {proportion, 0}, {border, 5}]), + wxSizer:add(Sizer, HSizer2, [{flag, ?wxEXPAND bor BorderFlags bor ?wxBOTTOM}, + {proportion, 0}, {border, 5}]), + wxPanel:setSizer(Panel, Sizer), Timer = observer_lib:start_timer(Config, 10), {Panel, #sys_wx_state{parent=Parent, parent_notebook=Notebook, panel=Panel, sizer=Sizer, - timer=Timer, fields=Fields0 ++ Fields1++Fields2++Fields3}}. + timer=Timer, fields=Fields0 ++ Fields1++Fields2++Fields3++Fields4}}. + create_sys_menu(Parent) -> View = {"View", [#create_menu{id = ?ID_REFRESH, text = "Refresh\tCtrl-R"}, @@ -83,14 +92,40 @@ create_sys_menu(Parent) -> update_syspage(#sys_wx_state{node = undefined}) -> ignore; update_syspage(#sys_wx_state{node = Node, fields=Fields, sizer=Sizer}) -> SysInfo = observer_wx:try_rpc(Node, observer_backend, sys_info, []), - {Sys, Mem, Cpu, Stats} = info_fields(), + {Sys, Mem, Cpu, Stats, Limits} = info_fields(), observer_lib:update_info(Fields, observer_lib:fill_info(Sys, SysInfo) ++ observer_lib:fill_info(Mem, SysInfo) ++ observer_lib:fill_info(Cpu, SysInfo) ++ - observer_lib:fill_info(Stats, SysInfo)), + observer_lib:fill_info(Stats, SysInfo)++ + observer_lib:fill_info(Limits, SysInfo)), + wxSizer:layout(Sizer). + +maybe_convert(undefined) -> "Not available"; +maybe_convert(V) -> observer_lib:to_str(V). + +get_dist_buf_busy_limit_info() -> + fun(Data) -> + maybe_convert(proplists:get_value(dist_buf_busy_limit, Data)) + end. + +get_limit_count_info(Count, Limit) -> + fun(Data) -> + C = proplists:get_value(Count, Data), + L = proplists:get_value(Limit, Data), + lists:flatten( + io_lib:format("~s / ~s ~s", + [maybe_convert(C), maybe_convert(L), + if + C =:= undefined -> ""; + L =:= undefined -> ""; + true -> io_lib:format("(~s % used)",[observer_lib:to_str({trunc, (C / L) *100})]) + end])) + end. + + info_fields() -> Sys = [{"System and Architecture", [{"System Version", otp_release}, @@ -122,14 +157,20 @@ info_fields() -> ]}], Stats = [{"Statistics", right, [{"Up time", {time_ms, uptime}}, - {"Max Processes", process_limit}, - {"Processes", process_count}, {"Run Queue", run_queue}, {"IO Input", {bytes, io_input}}, {"IO Output", {bytes, io_output}} ]} ], - {Sys, Mem, Cpu, Stats}. + Limits = [{"System statistics / limit", + [{"Atoms", get_limit_count_info(atom_count, atom_limit)}, + {"Processes", get_limit_count_info(process_count, process_limit)}, + {"Ports", get_limit_count_info(port_count, port_limit)}, + {"ETS", get_limit_count_info(ets_count, ets_limit)}, + {"Distribution buffer busy limit", get_dist_buf_busy_limit_info()} + ]}], + {Sys, Mem, Cpu, Stats, Limits}. + %%%%%%%%%%%%%%%%%%%%%%% Callbacks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/observer/src/observer_traceoptions_wx.erl b/lib/observer/src/observer_traceoptions_wx.erl index 285c298c4b..fca1510527 100644 --- a/lib/observer/src/observer_traceoptions_wx.erl +++ b/lib/observer/src/observer_traceoptions_wx.erl @@ -618,7 +618,7 @@ create_styled_txtctrl(Parent) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/os_mon/src/disksup.erl b/lib/os_mon/src/disksup.erl index 492e4814da..aeec335ba7 100644 --- a/lib/os_mon/src/disksup.erl +++ b/lib/os_mon/src/disksup.erl @@ -32,7 +32,7 @@ terminate/2, code_change/3]). %% Other exports --export([format_status/2]). +-export([format_status/2, parse_df/2]). -record(state, {threshold, timeout, os, diskdata = [],port}). @@ -285,7 +285,7 @@ check_disk_space({unix, sunos4}, Port, Threshold) -> Result = my_cmd("df", Port), check_disks_solaris(skip_to_eol(Result), Threshold); check_disk_space({unix, darwin}, Port, Threshold) -> - Result = my_cmd("/bin/df -i -k -t ufs,hfs", Port), + Result = my_cmd("/bin/df -i -k -t ufs,hfs,apfs", Port), check_disks_susv3(skip_to_eol(Result), Threshold). % This code works for Linux and FreeBSD as well @@ -294,8 +294,8 @@ check_disks_solaris("", _Threshold) -> check_disks_solaris("\n", _Threshold) -> []; check_disks_solaris(Str, Threshold) -> - case io_lib:fread("~s~d~d~d~d%~s", Str) of - {ok, [_FS, KB, _Used, _Avail, Cap, MntOn], RestStr} -> + case parse_df(Str, posix) of + {ok, {KB, Cap, MntOn}, RestStr} -> if Cap >= Threshold -> set_alarm({disk_almost_full, MntOn}, []); @@ -308,14 +308,102 @@ check_disks_solaris(Str, Threshold) -> check_disks_solaris(skip_to_eol(Str),Threshold) end. +%% @private +%% @doc Predicate to take a word from the input string until a space or +%% a percent '%' sign (the Capacity field is followed by a %) +parse_df_is_not_space($ ) -> false; +parse_df_is_not_space($%) -> false; +parse_df_is_not_space(_) -> true. + +%% @private +%% @doc Predicate to take spaces away from string. Stops on a non-space +parse_df_is_space($ ) -> true; +parse_df_is_space(_) -> false. + +%% @private +%% @doc Predicate to consume remaining characters until end of line. +parse_df_is_not_eol($\r) -> false; +parse_df_is_not_eol($\n) -> false; +parse_df_is_not_eol(_) -> true. + +%% @private +%% @doc Trims leading non-spaces (the word) from the string then trims spaces. +parse_df_skip_word(Input) -> + Remaining = lists:dropwhile(fun parse_df_is_not_space/1, Input), + lists:dropwhile(fun parse_df_is_space/1, Remaining). + +%% @private +%% @doc Takes all non-spaces and then drops following spaces. +parse_df_take_word(Input) -> + {Word, Remaining0} = lists:splitwith(fun parse_df_is_not_space/1, Input), + Remaining1 = lists:dropwhile(fun parse_df_is_space/1, Remaining0), + {Word, Remaining1}. + +%% @private +%% @doc Takes all non-spaces and then drops the % after it and the spaces. +parse_df_take_word_percent(Input) -> + {Word, Remaining0} = lists:splitwith(fun parse_df_is_not_space/1, Input), + %% Drop the leading % or do nothing + Remaining1 = case Remaining0 of + [$% | R1] -> R1; + _ -> Remaining0 % Might be no % or empty list even + end, + Remaining2 = lists:dropwhile(fun parse_df_is_space/1, Remaining1), + {Word, Remaining2}. + +%% @private +%% @doc Given a line of 'df' POSIX/SUSv3 output split it into fields: +%% a string (mounted device), 4 integers (kilobytes, used, available +%% and capacity), skip % sign, (optionally for susv3 can also skip IUsed, IFree +%% and ICap% fields) then take remaining characters as the mount path +-spec parse_df(string(), posix | susv3) -> + {error, parse_df} | {ok, {integer(), integer(), integer()}, string()}. +parse_df(Input0, Flavor) -> + %% Format of Posix/Linux df output looks like Header + Lines + %% Filesystem 1024-blocks Used Available Capacity Mounted on + %% udev 2467108 0 2467108 0% /dev + Input1 = parse_df_skip_word(Input0), % skip device path field + {KbStr, Input2} = parse_df_take_word(Input1), % take Kb field + Input3 = parse_df_skip_word(Input2), % skip Used field + Input4 = parse_df_skip_word(Input3), % skip Avail field + + % take Capacity% field; drop a % sign following the capacity + {CapacityStr, Input5} = parse_df_take_word_percent(Input4), + + %% Format of OS X/SUSv3 df looks similar to POSIX but has 3 extra columns + %% Filesystem 1024-blocks Used Available Capacity iused ifree %iused Mounted + %% /dev/disk1 243949060 2380 86690680 65% 2029724 37555 0% / + Input6 = case Flavor of + posix -> Input5; + susv3 -> % there are 3 extra integers we want to skip + Input5a = parse_df_skip_word(Input5), % skip IUsed field + Input5b = parse_df_skip_word(Input5a), % skip IFree field + %% skip the value of ICap + '%' field + {_, Input5c} = parse_df_take_word_percent(Input5b), + Input5c + end, + + % path is the remaining string till end of line + {MountPath, Input7} = lists:splitwith(fun parse_df_is_not_eol/1, Input6), + % Trim the newlines + Remaining = lists:dropwhile(fun(X) -> not parse_df_is_not_eol(X) end, + Input7), + try + Kb = erlang:list_to_integer(KbStr), + Capacity = erlang:list_to_integer(CapacityStr), + {ok, {Kb, Capacity, MountPath}, Remaining} + catch error:badarg -> + {error, parse_df} + end. + % Parse per SUSv3 specification, notably recent OS X check_disks_susv3("", _Threshold) -> []; check_disks_susv3("\n", _Threshold) -> []; check_disks_susv3(Str, Threshold) -> - case io_lib:fread("~s~d~d~d~d%~d~d~d%~s", Str) of - {ok, [_FS, KB, _Used, _Avail, Cap, _IUsed, _IFree, _ICap, MntOn], RestStr} -> + case parse_df(Str, susv3) of + {ok, {KB, Cap, MntOn}, RestStr} -> if Cap >= Threshold -> set_alarm({disk_almost_full, MntOn}, []); diff --git a/lib/os_mon/test/disksup_SUITE.erl b/lib/os_mon/test/disksup_SUITE.erl index ad61985014..d7f2626160 100644 --- a/lib/os_mon/test/disksup_SUITE.erl +++ b/lib/os_mon/test/disksup_SUITE.erl @@ -30,7 +30,7 @@ -export([port/1]). -export([terminate/1, unavailable/1, restart/1]). -export([otp_5910/1]). --export([posix_only/1]). +-export([posix_only/1, parse_df_output_posix/1, parse_df_output_susv3/1]). init_per_suite(Config) when is_list(Config) -> ok = application:start(os_mon), @@ -59,7 +59,8 @@ suite() -> all() -> Bugs = [otp_5910], - Always = [api, config, alarm, port, posix_only, unavailable] ++ Bugs, + Always = [api, config, alarm, port, posix_only, unavailable, + parse_df_output_posix, parse_df_output_susv3] ++ Bugs, case test_server:os_type() of {unix, _OSname} -> Always; {win32, _OSname} -> Always; @@ -413,3 +414,36 @@ get_disk_data([{"none",0,0}=E]) -> [E]; get_disk_data([{_,_,0}|Es]) -> get_disk_data(Es); get_disk_data([E|Es]) -> [E|get_disk_data(Es)]; get_disk_data([]) -> []. + +%% @doc Test various expected inputs to 'df' command output (Linux/POSIX) +parse_df_output_posix(Config) when is_list(Config) -> + PosixHdr = "Filesystem 1K-blocks Used Available Use% Mounted on\n", + {error, _} = disksup:parse_df(PosixHdr, posix), + {error, _} = disksup:parse_df("", posix), + {error, _} = disksup:parse_df("\n\n", posix), + + %% Have a simple example with no funny spaces in mount path + Posix1 = "tmpfs 498048 7288 490760 2% /run\n", + {ok, {498048, 2, "/run"}, ""} = disksup:parse_df(Posix1, posix), + + %% Have a mount path with some spaces in it + Posix2 = "tmpfs 498048 7288 490760 2% /spaces 1 2\n", + {ok, {498048, 2, "/spaces 1 2"}, ""} = disksup:parse_df(Posix2, posix). + +%% @doc Test various expected inputs to 'df' command output (Darwin/SUSv3) +parse_df_output_susv3(Config) when is_list(Config) -> + DarwinHdr = "Filesystem 1024-blocks Used Available Capacity " ++ + "iused ifree %iused Mounted on", + {error, _} = disksup:parse_df(DarwinHdr, susv3), + {error, _} = disksup:parse_df("", susv3), + {error, _} = disksup:parse_df("\n\n", susv3), + + %% Have a simple example with no funny spaces in mount path + Darwin1 = "/dev/disk1 243949060 157002380 86690680 65% 2029724 " ++ + "4292937555 0% /\n", + {ok, {243949060, 65, "/"}, ""} = disksup:parse_df(Darwin1, susv3), + + %% Have a mount path with some spaces in it + Darwin2 = "/dev/disk1 243949060 157002380 86690680 65% 2029724 " ++ + "4292937555 0% /spaces 1 2\n", + {ok, {243949060, 65, "/spaces 1 2"}, ""} = disksup:parse_df(Darwin2, susv3). diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index 04966ffb9c..942203bd12 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -129,18 +129,31 @@ <p><c>| 'rsa_no_padding'</c></p> </item> + <tag><c>public_sign_options() =</c></tag> + <item><p><c>[{rsa_pad, rsa_sign_padding()} | {rsa_pss_saltlen, integer()}]</c></p></item> + + <tag><c>rsa_sign_padding() =</c></tag> + <item> + <p><c>'rsa_pkcs1_padding'</c></p> + <p><c>| 'rsa_pkcs1_pss_padding'</c></p> + </item> + <tag><c>digest_type() = </c></tag> <item><p>Union of <c>rsa_digest_type()</c>, <c>dss_digest_type()</c>, and <c>ecdsa_digest_type()</c>.</p></item> <tag><c>rsa_digest_type() = </c></tag> - <item><p><c>'md5' | 'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p></item> + <item><p><c>'md5' | 'ripemd160' | 'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p></item> <tag><c>dss_digest_type() = </c></tag> - <item><p><c>'sha'</c></p></item> + <item><p><c>'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p> + <p>Note that the actual supported dss_digest_type depends on the underlying crypto library. + In OpenSSL version >= 1.0.1 the listed digest are supported, while in 1.0.0 only + sha, sha224 and sha256 are supported. In version 0.9.8 only sha is supported.</p> + </item> <tag><c>ecdsa_digest_type() = </c></tag> - <item><p><c>'sha'| 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p></item> + <item><p><c>'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p></item> <tag><c>crl_reason() = </c></tag> <item> @@ -621,8 +634,8 @@ fun(OtpCert :: #'OTPCertificate'{}, <v>OTPCertificate = #'OTPCertificate'{}</v> <v>DPAndCRLs = [{DP::#'DistributionPoint'{}, {DerCRL::der_encoded(), CRL::#'CertificateList'{}}}] </v> <v>Options = proplists:proplist()</v> - <v>CRLStatus() = valid | {bad_cert, revocation_status_undetermined} | - {bad_cert, {revoked, crl_reason()}}</v> + <v>CRLStatus() = valid | {bad_cert, revocation_status_undetermined} | {bad_cert, {revocation_status_undetermined, + {bad_crls, Details::term()}}} | {bad_cert, {revoked, crl_reason()}}</v> </type> <desc> <p>Performs CRL validation. It is intended to be called from @@ -650,7 +663,7 @@ fun(OtpCert :: #'OTPCertificate'{}, <tag>{issuer_fun, fun()}</tag> <item> <p>The fun has the following type specification:</p> - + <code> fun(#'DistributionPoint'{}, #'CertificateList'{}, {rdnSequence,[#'AttributeTypeAndValue'{}]}, term()) -> @@ -660,7 +673,15 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, that has signed the CRL. </p> <code> fun(DP, CRL, Issuer, UserState) -> {ok, RootCert, CertChain}</code> - </item> + </item> + + <tag>{undetermined_details, boolean()}</tag> + <item> + <p>Defaults to false. When revocation status can not be + determined, and this option is set to true, details of why no + CRLs where accepted are included in the return value.</p> + </item> + </taglist> </desc> </func> @@ -795,6 +816,7 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <func> <name>sign(Msg, DigestType, Key) -> binary()</name> + <name>sign(Msg, DigestType, Key, Options) -> binary()</name> <fsummary>Creates a digital signature.</fsummary> <type> <v>Msg = binary() | {digest,binary()}</v> @@ -803,6 +825,7 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, digest.</d> <v>DigestType = rsa_digest_type() | dss_digest_type() | ecdsa_digest_type()</v> <v>Key = rsa_private_key() | dsa_private_key() | ec_private_key()</v> + <v>Options = public_sign_options()</v> </type> <desc> <p>Creates a digital signature.</p> @@ -895,6 +918,7 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <func> <name>verify(Msg, DigestType, Signature, Key) -> boolean()</name> + <name>verify(Msg, DigestType, Signature, Key, Options) -> boolean()</name> <fsummary>Verifies a digital signature.</fsummary> <type> <v>Msg = binary() | {digest,binary()}</v> @@ -903,6 +927,7 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <v>DigestType = rsa_digest_type() | dss_digest_type() | ecdsa_digest_type()</v> <v>Signature = binary()</v> <v>Key = rsa_public_key() | dsa_public_key() | ec_public_key()</v> + <v>Options = public_sign_options()</v> </type> <desc> <p>Verifies a digital signature.</p> diff --git a/lib/public_key/include/public_key.hrl b/lib/public_key/include/public_key.hrl index a1e7dd31bc..663e1856ac 100644 --- a/lib/public_key/include/public_key.hrl +++ b/lib/public_key/include/public_key.hrl @@ -70,7 +70,8 @@ reasons_mask, cert_status, interim_reasons_mask, - valid_ext + valid_ext, + details }). -record('ECPoint', { diff --git a/lib/public_key/src/pubkey_crl.erl b/lib/public_key/src/pubkey_crl.erl index 33bef91827..3621e9c0da 100644 --- a/lib/public_key/src/pubkey_crl.erl +++ b/lib/public_key/src/pubkey_crl.erl @@ -58,7 +58,8 @@ validate(OtpCert, OtherDPCRLs, DP, {DerCRL, CRL}, {DerDeltaCRL, DeltaCRL}, init_revokation_state() -> #revoke_state{reasons_mask = sets:new(), interim_reasons_mask = sets:new(), - cert_status = unrevoked}. + cert_status = unrevoked, + details = []}. fresh_crl(_, {undefined, undefined}, _) -> %% Typically happens when there is no delta CRL that covers a CRL @@ -152,9 +153,10 @@ verify_crl(OtpCert, DP, CRL, DerCRL, DeltaCRL, DerDeltaCRL, OtherDPCRLs, RevokedState, CRL, DerCRL, DeltaCRL, DerDeltaCRL, IssuerFun, TrustedOtpCert, Path, OtherDPCRLs, IDP); - _ -> - {invalid, State0#revoke_state{valid_ext = ValidExt}} - end; + _ -> + Details = RevokedState#revoke_state.details, + {invalid, RevokedState#revoke_state{valid_ext = ValidExt, details = [{{bad_crl, no_issuer_cert_chain}, CRL} | Details]}} + end; {error, issuer_not_found} -> case Fun(DP, CRL, issuer_not_found, AdditionalArgs) of {ok, TrustedOtpCert, Path} -> @@ -163,13 +165,16 @@ verify_crl(OtpCert, DP, CRL, DerCRL, DeltaCRL, DerDeltaCRL, OtherDPCRLs, DerDeltaCRL, IssuerFun, TrustedOtpCert, Path, OtherDPCRLs, IDP); _ -> - {invalid, {skip, State0}} - end + Details = State0#revoke_state.details, + {invalid, {skip, State0#revoke_state{details = [{{bad_crl, no_issuer_cert_chain}, CRL} | Details] }}} + end catch - throw:{bad_crl, invalid_issuer} -> - {invalid, {skip, State0}}; - throw:_ -> - {invalid, State0#revoke_state{valid_ext = ValidExt}} + throw:{bad_crl, invalid_issuer} = Reason -> + Details = RevokedState#revoke_state.details, + {invalid, {skip, RevokedState#revoke_state{details = [{Reason, CRL} | Details]}}}; + throw:Reason -> + Details = RevokedState#revoke_state.details, + {invalid, RevokedState#revoke_state{details = [{Reason, CRL} | Details]}} end. verify_mask_and_signatures(Revoked, DeltaRevoked, RevokedState, CRL, DerCRL, DeltaCRL, DerDeltaCRL, @@ -183,10 +188,12 @@ verify_mask_and_signatures(Revoked, DeltaRevoked, RevokedState, CRL, DerCRL, Del TrustedOtpCert, Path, IssuerFun, OtherDPCRLs, IDP), {valid, Revoked, DeltaRevoked, RevokedState#revoke_state{reasons_mask = ReasonsMask}, IDP} catch - throw:_ -> - {invalid, RevokedState}; + throw:Reason -> + Details = RevokedState#revoke_state.details, + {invalid, RevokedState#revoke_state{details = [{Reason, CRL} | Details]}}; error:{badmatch, _} -> - {invalid, RevokedState} + Details = RevokedState#revoke_state.details, + {invalid, RevokedState#revoke_state{details = [{{bad_crl, invalid_signature}, CRL} | Details]}} end. @@ -356,7 +363,7 @@ verify_scope(#'OTPCertificate'{tbsCertificate = TBSCert}, #'DistributionPoint'{c verify_scope(DPName, IDPName, Names, TBSCert, IDP). verify_scope(asn1_NOVALUE, _, asn1_NOVALUE, _, _) -> - throw({bad_crl, scope_error1}); + throw({bad_crl, scope_error}); verify_scope(asn1_NOVALUE, IDPName, DPIssuerNames, TBSCert, IDP) -> verify_dp_name(IDPName, DPIssuerNames), verify_dp_bools(TBSCert, IDP); diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index 6651e9510e..c2060c144c 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -37,7 +37,7 @@ decrypt_public/2, decrypt_public/3, dh_gex_group/4, dh_gex_group_sizes/0, - sign/3, verify/4, + sign/3, sign/4, verify/4, verify/5, generate_key/1, compute_key/2, compute_key/3, pkix_sign/2, pkix_verify/2, @@ -90,10 +90,12 @@ auth_keys. -type rsa_padding() :: 'rsa_pkcs1_padding' | 'rsa_pkcs1_oaep_padding' | 'rsa_no_padding'. +-type rsa_sign_padding() :: 'rsa_pkcs1_padding' | 'rsa_pkcs1_pss_padding'. -type public_crypt_options() :: [{rsa_pad, rsa_padding()}]. --type rsa_digest_type() :: 'md5' | 'sha'| 'sha224' | 'sha256' | 'sha384' | 'sha512'. --type dss_digest_type() :: 'none' | 'sha'. %% None is for backwards compatibility --type ecdsa_digest_type() :: 'sha'| 'sha224' | 'sha256' | 'sha384' | 'sha512'. +-type rsa_digest_type() :: 'md5' | 'ripemd160' | 'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'. +-type dss_digest_type() :: 'none' | 'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'. %% None is for backwards compatibility +-type ecdsa_digest_type() :: 'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'. +-type public_sign_options() :: [{rsa_pad, rsa_sign_padding()} | {rsa_pss_saltlen, integer()}]. -type digest_type() :: rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(). -type crl_reason() :: unspecified | keyCompromise | cACompromise | affiliationChanged | superseded | cessationOfOperation | certificateHold | privilegeWithdrawn | aACompromise. @@ -417,7 +419,7 @@ generate_key({rsa, ModulusSize, PublicExponent}) -> {[E, N], [E, N, D, P, Q, D_mod_P_1, D_mod_Q_1, InvQ_mod_P]} -> Nint = crypto:bytes_to_integer(N), Eint = crypto:bytes_to_integer(E), - #'RSAPrivateKey'{version = 0, % Two-factor (I guess since otherPrimeInfos is not given) + #'RSAPrivateKey'{version = 'two-prime', % Two-factor (I guess since otherPrimeInfos is not given) modulus = Nint, publicExponent = Eint, privateExponent = crypto:bytes_to_integer(D), @@ -435,7 +437,7 @@ generate_key({rsa, ModulusSize, PublicExponent}) -> % 1976. Nint = crypto:bytes_to_integer(N), Eint = crypto:bytes_to_integer(E), - #'RSAPrivateKey'{version = 0, % Two-factor (I guess since otherPrimeInfos is not given) + #'RSAPrivateKey'{version = 'two-prime', % Two-factor (I guess since otherPrimeInfos is not given) modulus = Nint, publicExponent = Eint, privateExponent = crypto:bytes_to_integer(D), @@ -498,35 +500,67 @@ pkix_sign_types(?'ecdsa-with-SHA512') -> {sha512, ecdsa}. %%-------------------------------------------------------------------- --spec sign(binary() | {digest, binary()}, rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), - rsa_private_key() | - dsa_private_key() | ec_private_key()) -> Signature :: binary(). -%% Description: Create digital signature. -%%-------------------------------------------------------------------- -sign(DigestOrPlainText, DigestType, Key = #'RSAPrivateKey'{}) -> - crypto:sign(rsa, DigestType, DigestOrPlainText, format_rsa_private_key(Key)); +-spec sign(binary() | {digest, binary()}, + rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), + rsa_private_key() | dsa_private_key() | ec_private_key() + ) -> Signature :: binary(). -sign(DigestOrPlainText, sha, #'DSAPrivateKey'{p = P, q = Q, g = G, x = X}) -> - crypto:sign(dss, sha, DigestOrPlainText, [P, Q, G, X]); +-spec sign(binary() | {digest, binary()}, + rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), + rsa_private_key() | dsa_private_key() | ec_private_key(), + public_sign_options() + ) -> Signature :: binary(). -sign(DigestOrPlainText, DigestType, #'ECPrivateKey'{privateKey = PrivKey, - parameters = Param}) -> - ECCurve = ec_curve_spec(Param), - crypto:sign(ecdsa, DigestType, DigestOrPlainText, [PrivKey, ECCurve]); +%% Description: Create digital signature. +%%-------------------------------------------------------------------- +sign(DigestOrPlainText, DigestType, Key) -> + sign(DigestOrPlainText, DigestType, Key, []). %% Backwards compatible -sign(Digest, none, #'DSAPrivateKey'{} = Key) -> - sign({digest,Digest}, sha, Key). +sign(Digest, none, Key = #'DSAPrivateKey'{}, Options) when is_binary(Digest) -> + sign({digest, Digest}, sha, Key, Options); +sign(DigestOrPlainText, DigestType, Key, Options) -> + case format_sign_key(Key) of + badarg -> + erlang:error(badarg, [DigestOrPlainText, DigestType, Key, Options]); + {Algorithm, CryptoKey} -> + crypto:sign(Algorithm, DigestType, DigestOrPlainText, CryptoKey, Options) + end. %%-------------------------------------------------------------------- --spec verify(binary() | {digest, binary()}, rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), - Signature :: binary(), rsa_public_key() - | dsa_public_key() | ec_public_key()) -> boolean(). +-spec verify(binary() | {digest, binary()}, + rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), + Signature :: binary(), + rsa_public_key() | dsa_public_key() | ec_public_key() + | rsa_private_key() | dsa_private_key() | ec_private_key() + ) -> boolean(). + +-spec verify(binary() | {digest, binary()}, + rsa_digest_type() | dss_digest_type() | ecdsa_digest_type(), + Signature :: binary(), + rsa_public_key() | dsa_public_key() | ec_public_key() + | rsa_private_key() | dsa_private_key() | ec_private_key(), + public_sign_options() + ) -> boolean(). + %% Description: Verifies a digital signature. %%-------------------------------------------------------------------- -verify(DigestOrPlainText, DigestType, Signature, Key) when is_binary(Signature) -> - do_verify(DigestOrPlainText, DigestType, Signature, Key); -verify(_,_,_,_) -> +verify(DigestOrPlainText, DigestType, Signature, Key) -> + verify(DigestOrPlainText, DigestType, Signature, Key, []). + +%% Backwards compatible +verify(Digest, none, Signature, Key = {_, #'Dss-Parms'{}}, Options) when is_binary(Digest) -> + verify({digest, Digest}, sha, Signature, Key, Options); +verify(Digest, none, Signature, Key = #'DSAPrivateKey'{}, Options) when is_binary(Digest) -> + verify({digest, Digest}, sha, Signature, Key, Options); +verify(DigestOrPlainText, DigestType, Signature, Key, Options) when is_binary(Signature) -> + case format_verify_key(Key) of + badarg -> + erlang:error(badarg, [DigestOrPlainText, DigestType, Signature, Key, Options]); + {Algorithm, CryptoKey} -> + crypto:verify(Algorithm, DigestType, DigestOrPlainText, Signature, CryptoKey, Options) + end; +verify(_,_,_,_,_) -> %% If Signature is a bitstring and not a binary we know already at this %% point that the signature is invalid. false. @@ -789,8 +823,9 @@ pkix_path_validation(#'OTPCertificate'{} = TrustedCert, CertChain, Options) %-------------------------------------------------------------------- -spec pkix_crls_validate(#'OTPCertificate'{}, [{DP::#'DistributionPoint'{}, {DerCRL::binary(), CRL::#'CertificateList'{}}}], - Options :: proplists:proplist()) -> valid | {bad_cert, revocation_status_undetermined} - | {bad_cert, {revoked, crl_reason()}}. + Options :: proplists:proplist()) -> valid | {bad_cert, revocation_status_undetermined} | + {bad_cert, {revocation_status_undetermined, Reason::term()}} | + {bad_cert, {revoked, crl_reason()}}. %% Description: Performs a CRL validation according to RFC 5280. %%-------------------------------------------------------------------- @@ -993,22 +1028,32 @@ short_name_hash({rdnSequence, _Attributes} = Name) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -do_verify(DigestOrPlainText, DigestType, Signature, - #'RSAPublicKey'{modulus = Mod, publicExponent = Exp}) -> - crypto:verify(rsa, DigestType, DigestOrPlainText, Signature, - [Exp, Mod]); - -do_verify(DigestOrPlaintext, DigestType, Signature, {#'ECPoint'{point = Point}, Param}) -> - ECCurve = ec_curve_spec(Param), - crypto:verify(ecdsa, DigestType, DigestOrPlaintext, Signature, [Point, ECCurve]); - -%% Backwards compatibility -do_verify(Digest, none, Signature, {_, #'Dss-Parms'{}} = Key ) -> - verify({digest,Digest}, sha, Signature, Key); - -do_verify(DigestOrPlainText, sha = DigestType, Signature, {Key, #'Dss-Parms'{p = P, q = Q, g = G}}) - when is_integer(Key), is_binary(Signature) -> - crypto:verify(dss, DigestType, DigestOrPlainText, Signature, [P, Q, G, Key]). +format_sign_key(Key = #'RSAPrivateKey'{}) -> + {rsa, format_rsa_private_key(Key)}; +format_sign_key(#'DSAPrivateKey'{p = P, q = Q, g = G, x = X}) -> + {dss, [P, Q, G, X]}; +format_sign_key(#'ECPrivateKey'{privateKey = PrivKey, parameters = Param}) -> + {ecdsa, [PrivKey, ec_curve_spec(Param)]}; +format_sign_key(_) -> + badarg. + +format_verify_key(#'RSAPublicKey'{modulus = Mod, publicExponent = Exp}) -> + {rsa, [Exp, Mod]}; +format_verify_key({#'ECPoint'{point = Point}, Param}) -> + {ecdsa, [Point, ec_curve_spec(Param)]}; +format_verify_key({Key, #'Dss-Parms'{p = P, q = Q, g = G}}) -> + {dss, [P, Q, G, Key]}; +%% Convert private keys to public keys +format_verify_key(#'RSAPrivateKey'{modulus = Mod, publicExponent = Exp}) -> + format_verify_key(#'RSAPublicKey'{modulus = Mod, publicExponent = Exp}); +format_verify_key(#'ECPrivateKey'{parameters = Param, publicKey = {_, Point}}) -> + format_verify_key({#'ECPoint'{point = Point}, Param}); +format_verify_key(#'ECPrivateKey'{parameters = Param, publicKey = Point}) -> + format_verify_key({#'ECPoint'{point = Point}, Param}); +format_verify_key(#'DSAPrivateKey'{y=Y, p=P, q=Q, g=G}) -> + format_verify_key({Y, #'Dss-Parms'{p=P, q=Q, g=G}}); +format_verify_key(_) -> + badarg. do_pem_entry_encode(Asn1Type, Entity, CipherInfo, Password) -> Der = der_encode(Asn1Type, Entity), @@ -1121,8 +1166,13 @@ der_cert(#'OTPCertificate'{} = Cert) -> der_cert(Der) when is_binary(Der) -> Der. -pkix_crls_validate(_, [],_, _, _) -> - {bad_cert, revocation_status_undetermined}; +pkix_crls_validate(_, [],_, Options, #revoke_state{details = Details}) -> + case proplists:get_value(undetermined_details, Options, false) of + false -> + {bad_cert, revocation_status_undetermined}; + true -> + {bad_cert, {revocation_status_undetermined, {bad_crls, format_details(Details)}}} + end; pkix_crls_validate(OtpCert, [{DP, CRL, DeltaCRL} | Rest], All, Options, RevokedState0) -> CallBack = proplists:get_value(update_crl, Options, fun(_, CurrCRL) -> CurrCRL @@ -1142,9 +1192,14 @@ pkix_crls_validate(OtpCert, [{DP, CRL, DeltaCRL} | Rest], All, Options, Revoked do_pkix_crls_validate(OtpCert, [{DP, CRL, DeltaCRL} | Rest], All, Options, RevokedState0) -> OtherDPCRLs = All -- [{DP, CRL, DeltaCRL}], case pubkey_crl:validate(OtpCert, OtherDPCRLs, DP, CRL, DeltaCRL, Options, RevokedState0) of - {undetermined, _, _} when Rest == []-> - {bad_cert, revocation_status_undetermined}; - {undetermined, _, RevokedState} when Rest =/= []-> + {undetermined, unrevoked, #revoke_state{details = Details}} when Rest == []-> + case proplists:get_value(undetermined_details, Options, false) of + false -> + {bad_cert, revocation_status_undetermined}; + true -> + {bad_cert, {revocation_status_undetermined, {bad_crls, Details}}} + end; + {undetermined, unrevoked, RevokedState} when Rest =/= []-> pkix_crls_validate(OtpCert, Rest, All, Options, RevokedState); {finished, unrevoked} -> valid; @@ -1417,3 +1472,7 @@ to_lower_ascii(C) -> C. to_string(S) when is_list(S) -> S; to_string(B) when is_binary(B) -> binary_to_list(B). +format_details([]) -> + no_relevant_crls; +format_details(Details) -> + Details. diff --git a/lib/reltool/src/reltool.erl b/lib/reltool/src/reltool.erl index f6ce5578bc..feb6925044 100644 --- a/lib/reltool/src/reltool.erl +++ b/lib/reltool/src/reltool.erl @@ -80,7 +80,7 @@ get_server(WinPid) -> {ok, _ServerPid} = OK -> OK; {error, Reason} -> - {error, lists:flatten(io_lib:format("~p", [Reason]))} + {error, lists:flatten(io_lib:format("~tp", [Reason]))} end. %% Stop a server or window process @@ -93,7 +93,7 @@ stop(Pid) when is_pid(Pid) -> {'DOWN', Ref, _, _, shutdown} -> ok; {'DOWN', Ref, _, _, Reason} -> - {error, lists:flatten(io_lib:format("~p", [Reason]))} + {error, lists:flatten(io_lib:format("~tp", [Reason]))} end. %% Internal library function diff --git a/lib/reltool/src/reltool.hrl b/lib/reltool/src/reltool.hrl index 8b4898570b..9c8aae6b7e 100644 --- a/lib/reltool/src/reltool.hrl +++ b/lib/reltool/src/reltool.hrl @@ -119,7 +119,7 @@ | {archive, base_file(), [archive_opt()], [target_spec()]} | {copy_file, base_file()} | {copy_file, base_file(), top_file()} - | {write_file, base_file(), iolist()} + | {write_file, base_file(), binary()} | {strip_beam_file, base_file()}. -type target_dir() :: dir(). -type incl_defaults() :: boolean(). diff --git a/lib/reltool/src/reltool_app_win.erl b/lib/reltool/src/reltool_app_win.erl index 468ba297bb..663144861f 100644 --- a/lib/reltool/src/reltool_app_win.erl +++ b/lib/reltool/src/reltool_app_win.erl @@ -174,7 +174,7 @@ loop(#state{xref_pid = Xref, common = C, app = App} = S) -> S#state.mod_wins)}, ?MODULE:loop(S2); Msg -> - error_logger:format("~w~w got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~tp\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end. @@ -182,7 +182,7 @@ loop(#state{xref_pid = Xref, common = C, app = App} = S) -> exit_warning({'EXIT', _Pid, shutdown}) -> ok; exit_warning({'EXIT', _Pid, _Reason} = Msg) -> - error_logger:format("~w~w got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~tp\n", [?MODULE, self(), Msg]). create_window(#state{app = App} = S) -> @@ -629,7 +629,7 @@ handle_event(#state{sys = Sys, app = App} = S, Wx) -> handle_mod_button(S, Items, Action); _ -> error_logger:format("~w~w got unexpected app event from " - "wx:\n\t~p\n", + "wx:\n\t~tp\n", [?MODULE, self(), Wx]), S end. @@ -676,7 +676,7 @@ move_mod(App, {_ItemNo, ModStr}, Action) -> undefined; _ -> error_logger:format("~w~w got unexpected mod " - "button event: ~w\n\t ~p\n", + "button event: ~w\n\t ~tp\n", [?MODULE, self(), ModName, Action]), M#mod.incl_cond end, diff --git a/lib/reltool/src/reltool_fgraph_win.erl b/lib/reltool/src/reltool_fgraph_win.erl index 915330794c..a10a2281db 100644 --- a/lib/reltool/src/reltool_fgraph_win.erl +++ b/lib/reltool/src/reltool_fgraph_win.erl @@ -526,7 +526,7 @@ loop(S, G) -> exit(Reason); Other -> - error_logger:format("~w~w got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~tp\n", [?MODULE, self(), Other]), loop(S, G) end. diff --git a/lib/reltool/src/reltool_mod_win.erl b/lib/reltool/src/reltool_mod_win.erl index 8cd63bdda1..894d6e2ecb 100644 --- a/lib/reltool/src/reltool_mod_win.erl +++ b/lib/reltool/src/reltool_mod_win.erl @@ -171,7 +171,7 @@ loop(#state{xref_pid = Xref, common = C, mod = Mod} = S) -> S2 = handle_event(S, Wx), ?MODULE:loop(S2); _ -> - error_logger:format("~w~w got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~tp\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end @@ -487,7 +487,7 @@ handle_event(#state{xref_pid = Xref} = S, Wx) -> S; _ -> error_logger:format("~w~w got unexpected mod event from " - "wx:\n\t~p\n", + "wx:\n\t~tp\n", [?MODULE, self(), Wx]), S end. @@ -667,7 +667,7 @@ goto_function(S, Editor) -> wxStyledTextCtrl:setSelection(Editor, Left2, Right2), Text = wxStyledTextCtrl:getSelectedText(Editor), S2 = add_pos_to_history(S, CurrentPos), - do_goto_function(S2, string:tokens(Text, ":")); + do_goto_function(S2, string:lexemes(Text, ":")); _ -> %% No function call wxStyledTextCtrl:hideSelection(Editor, false), @@ -833,7 +833,7 @@ load_code(Ed, Code) when is_binary(Code) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/reltool/src/reltool_server.erl b/lib/reltool/src/reltool_server.erl index 89e90670cf..853191c696 100644 --- a/lib/reltool/src/reltool_server.erl +++ b/lib/reltool/src/reltool_server.erl @@ -225,12 +225,12 @@ parse_options([{Key, Val} | KeyVals], S, C, Sys) -> Sys2 = read_config(Sys, {sys, Val}), parse_options(KeyVals, S, C, Sys2); _ -> - reltool_utils:throw_error("Illegal option: ~p", [{Key, Val}]) + reltool_utils:throw_error("Illegal option: ~tp", [{Key, Val}]) end; parse_options([], S, C, Sys) -> S#state{common = C, sys = Sys}; parse_options(KeyVals, _S, _C, _Sys) -> - reltool_utils:throw_error("Illegal option: ~p", [KeyVals]). + reltool_utils:throw_error("Illegal option: ~tp", [KeyVals]). loop(#state{sys = Sys} = S) -> receive @@ -400,12 +400,12 @@ loop(#state{sys = Sys} = S) -> {'EXIT', Pid, Reason} when Pid =:= S#state.parent_pid -> exit(Reason); {call, ReplyTo, Ref, Msg} when is_pid(ReplyTo), is_reference(Ref) -> - error_logger:format("~w~w got unexpected call:\n\t~p\n", + error_logger:format("~w~w got unexpected call:\n\t~tp\n", [?MODULE, self(), Msg]), reltool_utils:reply(ReplyTo, Ref, {error, {invalid_call, Msg}}), ?MODULE:loop(S); Msg -> - error_logger:format("~w~w got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~tp\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end. @@ -1232,7 +1232,7 @@ parse_app_info(File, [{Key, Val} | KeyVals], AI, Status) -> Status); _ -> Status2 = - reltool_utils:add_warning("Unexpected item ~p in app file ~tp.", + reltool_utils:add_warning("Unexpected item ~tp in app file ~tp.", [Key,File], Status), parse_app_info(File, KeyVals, AI, Status2) @@ -1417,9 +1417,12 @@ shrink_app(A) -> do_save_config(S, Filename, InclDef, InclDeriv) -> {ok, Config} = do_get_config(S, InclDef, InclDeriv), - IoList = io_lib:format("%% config generated at ~w ~w\n~p.\n\n", - [date(), time(), Config]), - file:write_file(Filename, IoList). + IoList = io_lib:format("%% ~s\n" + "%% config generated at ~w ~w\n" + "~tp.\n\n", + [epp:encoding_to_string(utf8),date(), time(), Config]), + Bin = unicode:characters_to_binary(IoList), + file:write_file(Filename, Bin). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1455,7 +1458,7 @@ read_config(OldSys, {sys, KeyVals}) -> [NewSys2#sys.boot_rel]) end; read_config(_OldSys, BadConfig) -> - reltool_utils:throw_error("Illegal content: ~p", [BadConfig]). + reltool_utils:throw_error("Illegal content: ~tp", [BadConfig]). decode(#sys{apps = Apps} = Sys, [{erts = Name, AppKeyVals} | SysKeyVals]) when is_atom(Name), is_list(AppKeyVals) -> @@ -1565,7 +1568,7 @@ decode(#sys{} = Sys, [{Key, Val} | KeyVals]) -> debug_info when Val =:= keep; Val =:= strip -> Sys#sys{debug_info = Val}; _ -> - reltool_utils:throw_error("Illegal option: ~p", [{Key, Val}]) + reltool_utils:throw_error("Illegal option: ~tp", [{Key, Val}]) end, decode(Sys3, KeyVals); decode(#app{} = App, [{Key, Val} | KeyVals]) -> @@ -1620,14 +1623,14 @@ decode(#app{} = App, [{Key, Val} | KeyVals]) -> active_dir = Dir, sorted_dirs = [Dir]}; false -> - reltool_utils:throw_error("Illegal lib dir for ~w: ~p", + reltool_utils:throw_error("Illegal lib dir for ~w: ~tp", [App#app.name, Val]) end; SelectVsn when SelectVsn=:=vsn; SelectVsn=:=lib_dir -> reltool_utils:throw_error("Mutual exclusive options " "'vsn' and 'lib_dir'",[]); _ -> - reltool_utils:throw_error("Illegal option: ~p", [{Key, Val}]) + reltool_utils:throw_error("Illegal option: ~tp", [{Key, Val}]) end, decode(App2, KeyVals); decode(#app{mods = Mods} = App, [{mod, Name, ModKeyVals} | AppKeyVals]) -> @@ -1641,7 +1644,7 @@ decode(#mod{} = Mod, [{Key, Val} | KeyVals]) -> debug_info when Val =:= keep; Val =:= strip -> Mod#mod{debug_info = Val}; _ -> - reltool_utils:throw_error("Illegal option: ~p", [{Key, Val}]) + reltool_utils:throw_error("Illegal option: ~tp", [{Key, Val}]) end, decode(Mod2, KeyVals); decode(#rel{rel_apps = RelApps} = Rel, [RelApp | KeyVals]) -> @@ -1666,12 +1669,12 @@ decode(#rel{rel_apps = RelApps} = Rel, [RelApp | KeyVals]) -> true -> decode(Rel#rel{rel_apps = RelApps ++ [RA]}, KeyVals); false -> - reltool_utils:throw_error("Illegal option: ~p", [RelApp]) + reltool_utils:throw_error("Illegal option: ~tp", [RelApp]) end; decode(Acc, []) -> Acc; decode(_Acc, KeyVal) -> - reltool_utils:throw_error("Illegal option: ~p", [KeyVal]). + reltool_utils:throw_error("Illegal option: ~tp", [KeyVal]). is_type(Type) -> case Type of @@ -1866,7 +1869,7 @@ escripts_to_apps([Escript | Escripts], Apps, Status) -> {ok, AF} -> AF; {error, Reason1} -> - reltool_utils:throw_error("Illegal escript ~tp: ~p", + reltool_utils:throw_error("Illegal escript ~tp: ~tp", [Escript,Reason1]) end, @@ -1950,7 +1953,7 @@ escripts_to_apps([Escript | Escripts], Apps, Status) -> Status2), escripts_to_apps(Escripts, Apps2, Status3); {error, Reason2} -> - reltool_utils:throw_error("Illegal escript ~tp: ~p", + reltool_utils:throw_error("Illegal escript ~tp: ~tp", [Escript,Reason2]) end; escripts_to_apps([], Apps, Status) -> @@ -2013,7 +2016,7 @@ init_escript_app(AppName, EscriptAppName, Dir, Info, Mods, Apps, Status) -> case lists:keymember(AppName, #app.name, Apps) of true -> reltool_utils:throw_error( - "~w: Application name clash. Escript ~tp contains application ~tp.", + "~w: Application name clash. Escript ~tp contains application ~w.", [AppName,Dir,AppName]); false -> {App2, Status} diff --git a/lib/reltool/src/reltool_sys_win.erl b/lib/reltool/src/reltool_sys_win.erl index ba0d90ef5f..92df270752 100644 --- a/lib/reltool/src/reltool_sys_win.erl +++ b/lib/reltool/src/reltool_sys_win.erl @@ -136,7 +136,7 @@ init(Options) -> do_init(Options) catch error:Reason -> - io:format("~p: ~p~n",[Reason, erlang:get_stacktrace()]), + io:format("~tp: ~tp~n",[Reason, erlang:get_stacktrace()]), exit({Reason, erlang:get_stacktrace()}) end. @@ -182,7 +182,7 @@ do_init([{safe_config, Safe}, {parent, Parent} | Options]) -> end. restart_server_safe_config(true,Parent,Reason) -> - io:format("~w(~w): <ERROR> ~p\n", [?MODULE, ?LINE, Reason]), + io:format("~w(~w): <ERROR> ~tp\n", [?MODULE, ?LINE, Reason]), proc_lib:init_ack(Parent, {error,Reason}); restart_server_safe_config(false,Parent,Reason) -> wx:new(), @@ -199,7 +199,7 @@ restart_server_safe_config(false,Parent,Reason) -> ?wxID_OK -> do_init([{safe_config,true},{parent,Parent},?safe_config]); ?wxID_CANCEL -> - io:format("~w(~w): <ERROR> ~p\n", [?MODULE, ?LINE, Reason]), + io:format("~w(~w): <ERROR> ~tp\n", [?MODULE, ?LINE, Reason]), proc_lib:init_ack(Parent,{error,Reason}) end. @@ -251,7 +251,7 @@ loop(S) -> ?MODULE:loop(S#state{warning_wins = WWs2}); false -> error_logger:format("~w~w got unexpected " - "message:\n\t~p\n", + "message:\n\t~tp\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end @@ -292,7 +292,7 @@ loop(S) -> S#state.app_wins), ?MODULE:loop(S#state{fgraph_wins = FWs, app_wins = AWs}); Msg -> - error_logger:format("~w~w got unexpected message:\n\t~p\n", + error_logger:format("~w~w got unexpected message:\n\t~tp\n", [?MODULE, self(), Msg]), ?MODULE:loop(S) end. @@ -316,7 +316,7 @@ handle_child_exit({'EXIT', Pid, _Reason} = Exit, FWs, AWs) -> msg_warning({'EXIT', _Pid, shutdown}, Type) when Type =/= unknown -> ok; msg_warning(Exit, Type) -> - error_logger:format("~w~w got unexpected message (~w):\n\t~p\n", + error_logger:format("~w~w got unexpected message (~w):\n\t~tp\n", [?MODULE, self(), Type, Exit]). create_window(S) -> @@ -1163,12 +1163,12 @@ handle_system_event(#state{sys = Sys} = S, do_set_sys(S#state{sys = Sys2}); handle_system_event(S, Event, ObjRef, UserData) -> error_logger:format("~w~w got unexpected wx sys event to ~p " - "with user data: ~p\n\t ~p\n", + "with user data: ~tp\n\t ~tp\n", [?MODULE, self(), ObjRef, UserData, Event]), S. handle_release_event(S, _Event, _ObjRef, UserData) -> - io:format("Release data: ~p\n", [UserData]), + io:format("Release data: ~tp\n", [UserData]), S. handle_source_event(S, @@ -1225,7 +1225,7 @@ handle_app_event(S, handle_app_button(S, Items, Action); handle_app_event(S, Event, ObjRef, UserData) -> error_logger:format("~w~w got unexpected wx app event to " - "~p with user data: ~p\n\t ~p\n", + "~p with user data: ~tp\n\t ~tp\n", [?MODULE, self(), ObjRef, UserData, Event]), S. @@ -1269,7 +1269,7 @@ move_app(S, {_ItemNo, AppBase}, Action) -> undefined; _ -> error_logger:format("~w~w got unexpected app " - "button event: ~p ~p\n", + "button event: ~tp ~tp\n", [?MODULE, self(), Action, AppBase]), OldApp#app.incl_cond end, @@ -1543,7 +1543,7 @@ check_and_refresh(S, Status) -> display_message(Reason, ?wxICON_ERROR), false; {error, Reason} -> - Msg = lists:flatten(io_lib:format("Error:\n\n~p\n", [Reason])), + Msg = lists:flatten(io_lib:format("Error:\n\n~tp\n", [Reason])), display_message(Msg, ?wxICON_ERROR), false end, diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl index 1615a3e9b7..1b1461178e 100644 --- a/lib/reltool/src/reltool_target.erl +++ b/lib/reltool/src/reltool_target.erl @@ -787,16 +787,20 @@ do_spec_rel_files(#rel{name = RelName} = Rel, Sys) -> {ok, BootBin} = gen_boot(Script), Date = date(), Time = time(), - RelIoList = io_lib:format("%% rel generated at ~w ~w\n~p.\n\n", + RelIoList = io_lib:format("%% rel generated at ~w ~w\n~tp.\n\n", [Date, Time, GenRel]), - ScriptIoList = io_lib:format("%% script generated at ~w ~w\n~p.\n\n", + ScriptIoList = io_lib:format("%% script generated at ~w ~w\n~tp.\n\n", [Date, Time, Script]), [ - {write_file, RelFile, RelIoList}, - {write_file, ScriptFile, ScriptIoList}, + {write_file, RelFile, to_utf8_bin_with_enc_comment(RelIoList)}, + {write_file, ScriptFile, to_utf8_bin_with_enc_comment(ScriptIoList)}, {write_file, BootFile, BootBin} ]. +to_utf8_bin_with_enc_comment(IoList) when is_list(IoList) -> + unicode:characters_to_binary("%% " ++ epp:encoding_to_string(utf8) ++ "\n" + ++ IoList). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Generate a complete target system %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1001,7 +1005,8 @@ spec_start_file(#sys{boot_rel = BootRelName, {value, Erts} = lists:keysearch(erts, #app.name, Apps), {value, BootRel} = lists:keysearch(BootRelName, #rel.name, Rels), Data = Erts#app.vsn ++ " " ++ BootRel#rel.vsn ++ "\n", - {BootRel#rel.vsn, {write_file, "start_erl.data", Data}}. + {BootRel#rel.vsn, {write_file, "start_erl.data", + unicode:characters_to_binary(Data)}}. lookup_spec(Prefix, Specs) -> lists:filter(fun(S) -> lists:prefix(Prefix, element(2, S)) end, Specs). @@ -1183,18 +1188,18 @@ spec_app_file(#app{name = Name, Info#app_info.modules)], App2 = App#app{info = Info#app_info{modules = ModNames}}, Contents = gen_app(App2), - AppIoList = io_lib:format("%% app generated at ~w ~w\n~p.\n\n", + AppIoList = io_lib:format("%% app generated at ~w ~w\n~tp.\n\n", [date(), time(), Contents]), - [{write_file, AppFilename, AppIoList}]; + [{write_file, AppFilename, to_utf8_bin_with_enc_comment(AppIoList)}]; all -> %% Include all included modules %% Generate new file ModNames = [M#mod.name || M <- Mods, M#mod.is_included], App2 = App#app{info = Info#app_info{modules = ModNames}}, Contents = gen_app(App2), - AppIoList = io_lib:format("%% app generated at ~w ~w\n~p.\n\n", + AppIoList = io_lib:format("%% app generated at ~w ~w\n~tp.\n\n", [date(), time(), Contents]), - [{write_file, AppFilename, AppIoList}] + [{write_file, AppFilename, to_utf8_bin_with_enc_comment(AppIoList)}] end. @@ -1285,7 +1290,7 @@ do_eval_spec({archive, Archive, Options, Files}, {ok, _} -> ok; {error, Reason} -> - reltool_utils:throw_error("create archive ~ts failed: ~p", + reltool_utils:throw_error("create archive ~ts failed: ~tp", [ArchiveFile, Reason]) end; do_eval_spec({copy_file, File}, _OrigSourceDir, SourceDir, TargetDir) -> @@ -1299,12 +1304,12 @@ do_eval_spec({copy_file, File, OldFile}, SourceFile = filename:join([OrigSourceDir, OldFile]), TargetFile = filename:join([TargetDir, File]), reltool_utils:copy_file(SourceFile, TargetFile); -do_eval_spec({write_file, File, IoList}, +do_eval_spec({write_file, File, Bin}, _OrigSourceDir, _SourceDir, TargetDir) -> TargetFile = filename:join([TargetDir, File]), - reltool_utils:write_file(TargetFile, IoList); + reltool_utils:write_file(TargetFile, Bin); do_eval_spec({strip_beam, File}, _OrigSourceDir, SourceDir, TargetDir) -> SourceFile = filename:join([SourceDir, File]), TargetFile = filename:join([TargetDir, File]), @@ -1336,7 +1341,7 @@ cleanup_spec({copy_file, File}, TargetDir) -> cleanup_spec({copy_file, NewFile, _OldFile}, TargetDir) -> TargetFile = filename:join([TargetDir, NewFile]), file:delete(TargetFile); -cleanup_spec({write_file, File, _IoList}, TargetDir) -> +cleanup_spec({write_file, File, _}, TargetDir) -> TargetFile = filename:join([TargetDir, File]), file:delete(TargetFile); cleanup_spec({strip_beam, File}, TargetDir) -> @@ -1406,7 +1411,7 @@ do_filter_spec(Path, ExclRegexps) -> Path2 = opt_join(Path, NewFile), match(Path2, InclRegexps, ExclRegexps); -do_filter_spec(Path, {write_file, File, _IoList}, InclRegexps, ExclRegexps) -> +do_filter_spec(Path, {write_file, File, _}, InclRegexps, ExclRegexps) -> Path2 = opt_join(Path, File), match(Path2, InclRegexps, ExclRegexps); do_filter_spec(Path, {strip_beam, File}, InclRegexps, ExclRegexps) -> @@ -1448,7 +1453,7 @@ do_install(RelName, TargetDir) -> RelDir = filename:join([TargetDir2, "releases"]), DataFile = filename:join([RelDir, "start_erl.data"]), Bin = reltool_utils:read_file(DataFile), - case string:tokens(binary_to_list(Bin), " \n") of + case string:lexemes(unicode:characters_to_list(Bin), " \n") of [ErlVsn, RelVsn | _] -> ErtsBinDir = filename:join([TargetDir2, "erts-" ++ ErlVsn, "bin"]), BinDir = filename:join([TargetDir2, "bin"]), @@ -1501,8 +1506,8 @@ subst_src_script(Script, SrcDir, DestDir, Vars, Opts) -> subst_file(Src, Dest, Vars, Opts) -> Bin = reltool_utils:read_file(Src), - Chars = subst(binary_to_list(Bin), Vars), - reltool_utils:write_file(Dest, Chars), + Chars = subst(unicode:characters_to_list(Bin), Vars), + reltool_utils:write_file(Dest, unicode:characters_to_binary(Chars)), case lists:member(preserve, Opts) of true -> FileInfo = reltool_utils:read_file_info(Src), diff --git a/lib/reltool/src/reltool_utils.erl b/lib/reltool/src/reltool_utils.erl index 60edc9f3ca..3891b5ae4d 100644 --- a/lib/reltool/src/reltool_utils.erl +++ b/lib/reltool/src/reltool_utils.erl @@ -55,7 +55,7 @@ root_dir() -> code:root_dir(). erl_libs() -> - string:tokens(os:getenv("ERL_LIBS", ""), ":;"). + string:lexemes(os:getenv("ERL_LIBS", ""), ":;"). lib_dirs(Dir) -> case erl_prim_loader:list_dir(Dir) of @@ -286,7 +286,7 @@ split_app_dir(Dir) -> {Name, Vsn} = split_app_name(Base), Vsn2 = try - [list_to_integer(N) || N <- string:tokens(Vsn, ".")] + [list_to_integer(N) || N <- string:lexemes(Vsn, ".")] catch _:_ -> Vsn @@ -427,7 +427,7 @@ scroll_size(ObjRef) -> safe_keysearch(Key, Pos, List, Mod, Line) -> case lists:keysearch(Key, Pos, List) of false -> - io:format("~w(~w): lists:keysearch(~p, ~w, ~p) -> false\n", + io:format("~w(~w): lists:keysearch(~tp, ~w, ~tp) -> false\n", [Mod, Line, Key, Pos, List]), erlang:error({Mod, Line, lists, keysearch, [Key, Pos, List]}); {value, Val} -> @@ -498,8 +498,8 @@ read_file(File) -> throw_error("read file ~ts: ~ts", [File, Text]) end. -write_file(File, IoList) -> - case file:write_file(File, IoList) of +write_file(File, Bin) -> + case file:write_file(File, Bin) of ok -> ok; {error, Reason} -> @@ -601,7 +601,7 @@ do_decode_regexps(Key, [Regexp | Regexps], Acc) -> Regexps, [#regexp{source = Regexp, compiled = MP} | Acc]); _ -> - Text = lists:flatten(io_lib:format("~p", [{Key, Regexp}])), + Text = lists:flatten(io_lib:format("~tp", [{Key, Regexp}])), throw({error, "Illegal option: " ++ Text}) end; do_decode_regexps(_Key, [], Acc) -> diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 2bfc174cae..d50994306b 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.12.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A faulty encoding comment was added when saving trace + patterns to file. This is now corrected.</p> + <p> + Own Id: OTP-14479</p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.12</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/runtime_tools/src/dbg.erl b/lib/runtime_tools/src/dbg.erl index e82f27896d..92938ed5c1 100644 --- a/lib/runtime_tools/src/dbg.erl +++ b/lib/runtime_tools/src/dbg.erl @@ -268,7 +268,7 @@ wtp(FileName) -> {error, Reason} -> {error, Reason}; {ok, File} -> - io:put_chars(File, "%% coding: utf8\n"), + io:format(File, "%% ~s\n", [epp:encoding_to_string(utf8)]), pt_doforall(fun ({_, Val}, _) when is_list(Val) -> io:format(File, "~tp.~n", [Val]); ({_, _}, _) -> diff --git a/lib/runtime_tools/src/observer_backend.erl b/lib/runtime_tools/src/observer_backend.erl index d36af257ce..7f0c1ac6e4 100644 --- a/lib/runtime_tools/src/observer_backend.erl +++ b/lib/runtime_tools/src/observer_backend.erl @@ -63,9 +63,7 @@ sys_info() -> end, {{_,Input},{_,Output}} = erlang:statistics(io), - [{process_count, erlang:system_info(process_count)}, - {process_limit, erlang:system_info(process_limit)}, - {uptime, element(1, erlang:statistics(wall_clock))}, + [{uptime, element(1, erlang:statistics(wall_clock))}, {run_queue, erlang:statistics(run_queue)}, {io_input, Input}, {io_output, Output}, @@ -86,7 +84,17 @@ sys_info() -> {thread_pool_size, erlang:system_info(thread_pool_size)}, {wordsize_internal, erlang:system_info({wordsize, internal})}, {wordsize_external, erlang:system_info({wordsize, external})}, - {alloc_info, alloc_info()} + {alloc_info, alloc_info()}, + {process_count, erlang:system_info(process_count)}, + {atom_limit, erlang:system_info(atom_limit)}, + {atom_count, erlang:system_info(atom_count)}, + {process_limit, erlang:system_info(process_limit)}, + {process_count, erlang:system_info(process_count)}, + {port_limit, erlang:system_info(port_limit)}, + {port_count, erlang:system_info(port_count)}, + {ets_limit, erlang:system_info(ets_limit)}, + {ets_count, length(ets:all())}, + {dist_buf_busy_limit, erlang:system_info(dist_buf_busy_limit)} | MemInfo]. alloc_info() -> diff --git a/lib/runtime_tools/test/dbg_SUITE.erl b/lib/runtime_tools/test/dbg_SUITE.erl index 4b0864858c..cfe8412e33 100644 --- a/lib/runtime_tools/test/dbg_SUITE.erl +++ b/lib/runtime_tools/test/dbg_SUITE.erl @@ -23,7 +23,7 @@ -export([all/0, suite/0, big/1, tiny/1, simple/1, message/1, distributed/1, port/1, send/1, recv/1, - ip_port/1, file_port/1, file_port2/1, file_port_schedfix/1, + ip_port/1, file_port/1, file_port2/1, ip_port_busy/1, wrap_port/1, wrap_port_time/1, with_seq_trace/1, dead_suspend/1, local_trace/1, saved_patterns/1, tracer_exit_on_stop/1, @@ -41,7 +41,7 @@ suite() -> all() -> [big, tiny, simple, message, distributed, port, ip_port, send, recv, - file_port, file_port2, file_port_schedfix, ip_port_busy, + file_port, file_port2, ip_port_busy, wrap_port, wrap_port_time, with_seq_trace, dead_suspend, local_trace, saved_patterns, tracer_exit_on_stop, erl_tracer, distributed_erl_tracer]. @@ -623,99 +623,6 @@ file_port2(Config) when is_list(Config) -> end, ok. -%% Test that the scheduling timestamp fix for trace flag 'running' works. -file_port_schedfix(Config) when is_list(Config) -> - case (catch erlang:system_info(smp_support)) of - true -> - {skip, "No schedule fix on SMP"}; - _ -> - try - file_port_schedfix1(Config) - after - dbg:stop() - end - end. -file_port_schedfix1(Config) when is_list(Config) -> - stop(), - {A,B,C} = erlang:now(), - FTMP = atom_to_list(?MODULE) ++ integer_to_list(A) ++ - "-" ++ integer_to_list(B) ++ "-" ++ integer_to_list(C), - FName = filename:join([proplists:get_value(data_dir, Config), FTMP]), - %% - Port = dbg:trace_port(file, {FName, wrap, ".wraplog", 8*1024, 4}), - {ok, _} = dbg:tracer(port, Port), - {ok,[{matched,_node,0}]} = dbg:p(new,[running,procs,send,timestamp]), - %% - %% Generate the trace data - %% - %% This starts 3 processes that sends a message to each other in a ring, - %% 4 laps. Prior to sending the message to the next in the ring, each - %% process send 8 messages to itself, just to generate some trace data, - %% and to lower the possibility that the trace log wraps just after - %% a schedule out message (which would not burden any process and hence - %% not show up in the result) - %% - %% The wrap file trace is used because it burns a lot of time when the - %% driver swaps files, a lot more than the regular file trace. The test - %% case is dimensioned so that the log fills two files and just starts - %% on the third (out of four wrap files). This gives two file swaps, - %% and there are three processes, so one process will NOT be burdened. - %% The criterion for trace success is then that the max process - %% execution time must not be more than twice the min process - %% execution time. Wallclock. A normal result is about 10 times more - %% without schedule in - schedule out compensation (OTP-3938). - %% - ok = token_volleyball(3, 4, 8), - %% - {ok,[{matched,_,_}]} = dbg:p(all, [clear]), - stop(), - %% - %% Get the trace result - %% - Tag = make_ref(), - dbg:trace_client(file, {FName, wrap, ".wraplog"}, - {fun schedstat_handler/2, {self(), Tag, []}}), - Result = - receive - {Tag, D} -> - lists:map( - fun({Pid, {A1, B1, C1}}) -> - {Pid, C1/1000000 + B1 + A1*1000000} - end, - D) - end, - ok = io:format("Result=~p", [Result]), - % erlang:display({?MODULE, ?LINE, Result}), - %% - %% Analyze the result - %% - {Min, Max} = lists:foldl(fun({_Pid, M}, {Mi, Ma}) -> - {if M < Mi -> M; true -> Mi end, - if M > Ma -> M; true -> Ma end} - end, - {void, 0}, - Result), - % More PaN debug - io:format("Min = ~f, Max = ~f~n",[Min,Max]), - %% - %% Cleanup - %% - ToBeDeleted = filelib:wildcard(FName++"*"++".wraplog"), - lists:map(fun file:delete/1, ToBeDeleted), - % io:format("ToBeDeleted=~p", [ToBeDeleted]), - %% - %% Present the result - %% - P = (Max / Min - 1) * 100, - BottomLine = lists:flatten(io_lib:format("~.2f %", [P])), - if P > 100 -> - Reason = {BottomLine, '>', "100%"}, - erlang:display({file_port_schedfix, fail, Reason}), - ct:fail(Reason); - true -> - {comment, BottomLine} - end. - %% Test tracing to wrapping file port wrap_port(Config) when is_list(Config) -> Self = self(), diff --git a/lib/runtime_tools/test/dyntrace_SUITE.erl b/lib/runtime_tools/test/dyntrace_SUITE.erl index 7be2f49a8b..7ffbe54446 100644 --- a/lib/runtime_tools/test/dyntrace_SUITE.erl +++ b/lib/runtime_tools/test/dyntrace_SUITE.erl @@ -51,11 +51,7 @@ init_per_suite(Config) -> case erlang:system_info(debug_compiled) of false -> ""; true -> ".debug" - end ++ - case erlang:system_info(smp_support) of - false -> ""; - true -> ".smp" - end, + end ++ ".smp", [{emu_name,N}|Config]. end_per_suite(_Config) -> diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index 5ee39a25fe..7296221033 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.12 +RUNTIME_TOOLS_VSN = 1.12.1 diff --git a/lib/sasl/src/Makefile b/lib/sasl/src/Makefile index ac7ee51100..45cd814bf8 100644 --- a/lib/sasl/src/Makefile +++ b/lib/sasl/src/Makefile @@ -37,7 +37,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/sasl-$(VSN) MODULES= alarm_handler sasl sasl_report \ sasl_report_file_h sasl_report_tty_h format_lib_supp \ misc_supp rb rb_format_supp release_handler \ - release_handler_1 si si_sasl_supp systools \ + release_handler_1 systools \ systools_make systools_rc systools_relup systools_lib \ erlsrv diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl index 1f3c6877d5..d0a7c7332d 100644 --- a/lib/sasl/src/release_handler.erl +++ b/lib/sasl/src/release_handler.erl @@ -1143,8 +1143,9 @@ new_emulator_make_hybrid_config(CurrentVsn,ToVsn,TmpVsn,RelDir,Masters) -> Config2 = replace_config(stdlib,Config1,Stdlib), Config3 = replace_config(sasl,Config2,Sasl), - ConfigStr = io_lib:format("~p.~n",[Config3]), - write_file(TmpFile,ConfigStr,Masters). + ConfigStr = io_lib:format("%% ~s~n~tp.~n", + [epp:encoding_to_string(utf8),Config3]), + write_file(TmpFile,unicode:characters_to_binary(ConfigStr),Masters). %% Take the configuration for application App from the new config and %% insert in the old config. @@ -1874,9 +1875,10 @@ write_releases_1(Dir, NewReleases, Masters) -> write_releases_m(Dir, NewReleases, Masters). do_write_release(Dir, RELEASES, NewReleases) -> - case file:open(filename:join(Dir, RELEASES), [write]) of + case file:open(filename:join(Dir, RELEASES), [write,{encoding,utf8}]) of {ok, Fd} -> - ok = io:format(Fd, "~p.~n", [NewReleases]), + ok = io:format(Fd, "%% ~s~n~tp.~n", + [epp:encoding_to_string(utf8),NewReleases]), ok = file:close(Fd); {error, Reason} -> {error, Reason} diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src index 633cdfa070..d75543a91b 100644 --- a/lib/sasl/src/sasl.app.src +++ b/lib/sasl/src/sasl.app.src @@ -32,8 +32,6 @@ sasl_report, sasl_report_tty_h, sasl_report_file_h, - si, - si_sasl_supp, systools, systools_make, systools_rc, diff --git a/lib/sasl/src/sasl.appup.src b/lib/sasl/src/sasl.appup.src index 7f866507a0..94af164b20 100644 --- a/lib/sasl/src/sasl.appup.src +++ b/lib/sasl/src/sasl.appup.src @@ -18,7 +18,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* + [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.* %% Down to - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* + [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* }. diff --git a/lib/sasl/src/si.erl b/lib/sasl/src/si.erl deleted file mode 100644 index 275c6d508b..0000000000 --- a/lib/sasl/src/si.erl +++ /dev/null @@ -1,169 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% -%%----------------------------------------------------------------- -%% l(format_lib_supp), l(si_sasl_supp), l(si), l(si_ms_aos_supp), l(misc_supp). -%% c(format_lib_supp), c(si_sasl_supp), c(si), c(si_ms_aos_supp), c(misc_supp). -%%----------------------------------------------------------------- - - -%%-------------------------------------------------- -%% Description: -%% Status Inspection, main module. -%%-------------------------------------------------- - --module(si). - - -%% External exports --export([h/0, help/0, start/0, start/1, start_log/1, stop_log/0, - abbrevs/0, pi/1, pi/2, pi/3, pi/4, ppi/1, ppi/3, stop/0]). - -%% Internal exports --export([pi_impl/2, test/0]). - - -%%-------------------------------------------------- -%% Table of contents -%% 1. Interface -%% 2. Implementation - - --import(si_sasl_supp, [status_info/1, make_pid/1, p/1]). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% 1. Interface -%%-------------------------------------------------- - -h() -> print_help(). -help() -> print_help(). - -start() -> si_sasl_supp:start(). -start(Options) -> si_sasl_supp:start(Options). - -stop() -> si_sasl_supp:stop(). - -start_log(FileName) -> - si_sasl_supp:start_log(FileName). - -stop_log() -> - si_sasl_supp:stop_log(). - -%%%----------------------------------------------------------------- -%%% All functions can be called with an option 'normal' or 'all'; -%%% default is 'normal'. -%%%----------------------------------------------------------------- - -abbrevs() -> - io:format("~p", [lists:append(si_sasl_supp:process_abbrevs(), - process_abbrevs())]). - -%%----------------------------------------------------------------- -%% Process Info that tries to determine processtype (=Module), then -%% it uses this Module:format_info to format data from status_info/1. -%%----------------------------------------------------------------- -pi(XPid) -> - si_sasl_supp:si_exec({si, pi_impl}, [normal, XPid]). - -pi(Opt, XPid) -> - si_sasl_supp:si_exec({si, pi_impl}, [si_sasl_supp:valid_opt(Opt), XPid]). - -pi(A, B, C) when is_integer(A), is_integer(B), is_integer(C) -> - si_sasl_supp:si_exec({si, pi_impl}, [normal, {A, B, C}]). - -pi(Opt, A, B, C) when is_integer(A), is_integer(B), is_integer(C) -> - si_sasl_supp:si_exec({si, pi_impl}, [si_sasl_supp:valid_opt(Opt), {A, B, C}]). - -%%----------------------------------------------------------------- -%% Pretty print Process_Info. -%%----------------------------------------------------------------- -ppi(XPid) -> - si_sasl_supp:ppi(XPid). -ppi(A, B, C) -> - si_sasl_supp:ppi(A, B, C). - - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% 2. Implementation -%%-------------------------------------------------- - -print_help() -> - p("~nStatus Inspection tool - usage"), - p("=============================="), - p(" For all these functions, Opt is an optional argument"), - p(" which can be 'normal' or 'all'; default is 'normal'."), - p(" If 'all', all information will be printed."), - p(" A Pid can be: \"<A.B.C>\", {A, B, C}, B, a registered_name or an abbrev."), - p("ANY PROCESS"), - p("si:pi([Opt,] Pid) - Formatted information about any process that"), - p(" SI recognises."), - p("si:pi([Opt,] A,B,C) - Same as si:pi({A, B, C})."), - p("si:ppi(Pid) - Pretty formating of process_info."), - p(" Works for any process."), - p("MISC"), - p("si:abbrevs() - Lists valid abbreviations."), - p("si:start_log(Filename) - Logging to file."), - p("si:stop_log()"), - p("si:start() - Starts Status Inspection (the si_server)."), - p("si:start([{start_log, FileName}])"), - p("si:stop() - Shut down SI."). - - -%%-------------------------------------------------- -%% Copied (and modified) code from si_sasl_supp. -%%-------------------------------------------------- -pi_impl(Opt, XPid) -> - case make_pid(try_local_expand_abbrev(XPid)) of - Pid when is_pid(Pid) -> - case status_info(Pid) of - {status_info, Pid, {module, Module}, Data} -> - si_sasl_supp:do_best_printout(Opt, Pid, Module, Data); - {error, Reason} -> - _ = si_sasl_supp:ppi_impl(Pid), - {error, {"can not get status info from process:", - XPid, - Reason}}; - Else -> - {error, {"unknown status info", Else}} - end; - {error, Reason} -> - {error, Reason} - end. - -%%-------------------------------------------------- -%% Functions for handling of abbreviations -%%-------------------------------------------------- -try_local_expand_abbrev(Abbrev) -> - case si_sasl_supp:expand_abbrev(Abbrev, process_abbrevs()) of - {value, {_, RealName}} -> RealName; - _ -> Abbrev - end. - -process_abbrevs() -> - []. - -%% Test get_status_info/format_status_info for all implemented servers. -test() -> - lists:foreach(fun test_all_registered/1, - lists:append(si_sasl_supp:process_abbrevs(), - process_abbrevs())). - -test_all_registered({Al, _Ful}) -> - si:pi(all, Al). diff --git a/lib/sasl/src/si_sasl_supp.erl b/lib/sasl/src/si_sasl_supp.erl deleted file mode 100644 index cce628f8c4..0000000000 --- a/lib/sasl/src/si_sasl_supp.erl +++ /dev/null @@ -1,380 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(si_sasl_supp). - --behaviour(gen_server). - -%%%--------------------------------------------------------------------------- -%%% Description: -%%% This module contains the BOS specific parts of the Status Inspection Tool. -%%%--------------------------------------------------------------------------- - - -%% user interface --export([h/0, help/0, start_log/1, stop_log/0, abbrevs/0, pi/1, pi/2, pi/3, - pi/4, ppi/1, ppi/3, start/0, start/1, stop/0, start_link/1]). - -%% intermodule exports --export([make_pid/1, make_pid/3, process_abbrevs/0, expand_abbrev/2, - status_info/1, valid_opt/1, p/1, do_best_printout/4, - si_exec/2, handle_call/3, terminate/2]). - -%% exports for use within module --export([init/1, start_log_impl/1, pi_impl/2, ppi_impl/1]). - -%% other gen_server callbacks (not used) --export([handle_cast/2, handle_info/2, code_change/3]). - -%%-------------------------------------------------- -%% Table of contents -%% 1. Interface -%% 2. SI - Server -%% 3. Code -%% 4. Selectors -%%-------------------------------------------------- - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% 1. Interface -%% ----------------------------------------------------- - -h() -> print_help(). -help() -> print_help(). - -si_exec(Fun, Args) -> call({si_exec, Fun, Args}). - -start_log(FileName) -> - call({start_log, FileName}). - -stop_log() -> - call(stop_log). - -abbrevs() -> - io:format("~p", [process_abbrevs()]). - -%%----------------------------------------------------------------- -%% All functions can be called with an option 'normal' or 'all'; -%% default is 'normal'. -%%----------------------------------------------------------------- -%% Process Info that tries to determine processtype (=Module), then -%% it uses this Module:format_info to format data from status_info/1. -%%----------------------------------------------------------------- -pi(XPid) -> - si_exec({si_sasl_supp, pi_impl}, [normal, XPid]). - -pi(Opt, XPid) -> - si_exec({si_sasl_supp, pi_impl}, [valid_opt(Opt), XPid]). - -pi(A, B, C) when is_integer(A), is_integer(B), is_integer(C) -> - si_exec({si_sasl_supp, pi_impl}, [normal, {A, B, C}]). - -pi(Opt, A, B, C) when is_integer(A), is_integer(B), is_integer(C) -> - si_exec({si_sasl_supp, pi_impl}, [valid_opt(Opt), {A, B, C}]). - -%%----------------------------------------------------------------- -%% Pretty print Process_Info. -%%----------------------------------------------------------------- -ppi(XPid) -> - case whereis(si_server) of - undefined -> % You can always run ppi. - ppi_impl(XPid); % if si_server is down, use standard_io - _ -> - si_exec({si_sasl_supp, ppi_impl}, [XPid]) - end. -ppi(A, B, C) -> - case whereis(si_server) of - undefined -> % You can always run ppi. - ppi_impl({A, B, C}); % if si_server is down, use standard_io - _ -> - si_exec({si_sasl_supp, ppi_impl}, [{A, B, C}]) - end. - - - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% 2. SI - Server -%%-------------------------------------------------- --record(state, {}). - -start() -> start([]). -start(Options) -> - supervisor:start_child(sasl_sup, - {si_server, {si_sasl_supp, start_link, [Options]}, - temporary, brutal_kill, worker, [si_sasl_supp]}). - -start_link(_Options) -> - gen_server:start_link({local, si_server}, si_sasl_supp, [], []). - -stop() -> - call(stop), - supervisor:delete_child(sasl_sup, si_server). - - -init(Options) -> - process_flag(trap_exit, true), - start_log_impl(get_option(Options, start_log, standard_io)), - {ok, #state{}}. - -%%----------------------------------------------------------------- -%% If an error occurs and we're logging to file: write the error -%% to the file. -%% Always return the error. -%% The only data held by the si_server is the device in its process dictionary. -%%----------------------------------------------------------------- -handle_call({si_exec, Fun, Args}, _From, State) -> - case catch apply(Fun, Args) of - {'EXIT', Reason} -> - print_error(get(device), - "SI internal error. Reason: ~w~n", - [Reason]), - {stop, shutdown, {internal_error, Reason}, State}; - {error, Reason} -> - print_error(get(device), "~nSI error: ~w~n", [Reason]), - {reply, {error, Reason}, State}; - X -> - {reply, X, State} - end; -handle_call({start_log, FileName}, _From, State) -> - start_log_impl(FileName), - {reply, ok, State}; -handle_call(stop_log, _From, State) -> - start_log_impl(standard_io), - {reply, ok, State}; -handle_call(stop, _From, State) -> - start_log_impl(standard_io), - {stop, normal, stopped, State}. - -terminate(_Reason, _State) -> - _ = close_device(get(device)), - ok. - -handle_cast(_Msg, State) -> - {noreply, State}. -handle_info(_Info, State) -> - {noreply, State}. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -close_device(standard_io) -> ok; -close_device(Fd) -> file:close(Fd). - -print_error(standard_io, _, _) -> ok; -print_error(Device, Format, Args) -> - io:format(Device, Format, Args). - -get_option(Options, Key, Default) -> - case lists:keysearch(Key, 1, Options) of - {value, {_Key, Value}} -> Value; - _ -> Default - end. - -open_log_file(undefined, NewFile) -> - open_log_file(NewFile); -open_log_file(standard_io, NewFile) -> - open_log_file(NewFile); -open_log_file(OldFile, NewFile) -> - _ = file:close(OldFile), - open_log_file(NewFile). - -open_log_file(standard_io) -> standard_io; -open_log_file(FileName) -> - case file:open(FileName, [write]) of - {ok, Fd} -> Fd; - Error -> - io:format("si_sasl_supp: Cannot open file '~s' (~w).~n", - [FileName, Error]), - io:format("si_sasl_supp: Using standard_io~n"), - standard_io - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% 3. Code -%%-------------------------------------------------- - -%%----------------------------------------------------------------- -%% call(Request) -> Term -%%----------------------------------------------------------------- -call(Req) -> - gen_server:call(si_server, Req, infinity). - -%%-------------------------------------------------- -%% Makes a Pid of almost anything. -%% Returns: Pid|{error, Reason} -%% Fails: Never. -%%-------------------------------------------------- -make_pid(A,B,C) when is_integer(A), is_integer(B), is_integer(C) -> - list_to_pid(lists:concat(["<",A,".",B,".",C,">"])). -make_pid(P) when is_pid(P) -> P; -make_pid(undefined) -> undefined; -make_pid(P) when is_atom(P) -> - case whereis(P) of - undefined -> - case expand_abbrev(P, process_abbrevs()) of - {error, Reason} -> {error, Reason}; - {value, {_Abbrev, FullName}} -> - case whereis(FullName) of - undefined -> - {error, {'process not registered', P}}; - Pid -> Pid - end - end; - Pid -> Pid - end; -make_pid(P) when is_list(P) -> list_to_pid(P); -make_pid({A, B, C}) -> make_pid(A, B, C); -make_pid(X) -> {error, {'can not make a pid of', X}}. - -process_abbrevs() -> - [{init, init}, - {fs, file_server}]. - -%%-------------------------------------------------- -%% Args: Abbrevs is an assoc-list of {Abbrev, RealName} -%% Returns: {value, {Abbrev, FullName}}|{error, Reason} -%%-------------------------------------------------- -expand_abbrev(ProcessName, Abbrevs) -> - case lists:keysearch(ProcessName, 1, Abbrevs) of - {value, {Abbrev, FullName}} -> - {value, {Abbrev, FullName}}; - _ -> - case lists:keysearch(ProcessName, 2, Abbrevs) of - {value, {Abbrev, FullName}} -> - {value, {Abbrev, FullName}}; - _ -> - {error, {'invalid process name', ProcessName}} - end - end. - -%%----------------------------------------------------------------- -%% This is the function that actually gets the information out -%% of the agent/server/... -%% Returns: {status_info, Pid, Type, Data} -%% | {error, Reason} -%%----------------------------------------------------------------- -status_info(Pid) when is_pid(Pid) -> - case catch sys:get_status(Pid, 5000) of - {status, Pid, Type, Info} -> - {status_info, Pid, Type, Info}; - _ -> - {error, {'process does not respond', Pid}} - end; - -status_info(X) -> - {error, {'not a pid', X}}. - -%%-------------------------------------------------- -%% Implementation starts here. -%%-------------------------------------------------- -start_log_impl(FileName) -> - put(device, open_log_file(get(device), FileName)). - -valid_opt(all) -> all; -valid_opt(_Opt) -> normal. - - -print_help() -> - p("- - - - - - - - PROCESSES - - - - - - - - - "), - p("si_sasl_supp:pi([Opt,] Pid) - Formatted information about any process that"), - p(" SI recognises."), - p("si_sasl_supp:pi([Opt,] A,B,C) - Same as si_sasl_supp:pi({A, B, C})."), - p("si_sasl_supp:ppi(Pid) - Pretty formating of process_info."), - p(" Works for any process."), - p("- - - - - - - - MISC - - - - - - - - - - - "), - p("si_sasl_supp:abbrevs() - Lists valid abbreviations."), - p("si_sasl_supp:start_log(FileNname)"), - p("si_sasl_supp:stop_log()"), - p("si_sasl_supp:start() - Starts Status Inspection (the si_server)."), - p("si_sasl_supp:start([{start_log, FileName}])"), - p("si_sasl_supp:stop() - Shut down SI."). - - - -%% Convenient shorthand -p(X) -> - io:format(lists:append(X, "~n")). - -pi_impl(Opt, XPid) -> - case make_pid(XPid) of - Pid when is_pid(Pid) -> - case status_info(Pid) of - {status_info, Pid, {module, Module}, Data} -> - do_best_printout(Opt, Pid, Module, Data); - {error, Reason} -> - _ = ppi_impl(Pid), - {error, {"can not get status info from process:", - XPid, - Reason}} - end; - {error, Reason} -> - {error, Reason} - end. - -%%-------------------------------------------------- -%% Is there a format_info for this process? In that case, run it! -%% Return ok|{error, Reason} -%% Fails: Never. -%%-------------------------------------------------- -do_best_printout(Opt, Pid, Mod, Data) when is_pid(Pid) -> - case print_info(get(device), Pid, {Mod, format_status}, Opt, Data) of - ok -> ok; - {error, Reason} -> - _ = ppi_impl(Pid), - {error, Reason} - end. - -ppi_impl(XPid) -> - case make_pid(XPid) of - P when is_pid(P) -> - case process_info(P) of - undefined -> - {error, {'dead process', P}}; - PI -> - Device = case get(device) of - undefined -> standard_io; - X -> X - end, - io:format(Device, "~nPretty Process Info~n", []), - io:format(Device, "-------------------~n", []), - io:format(Device, "~p~n", [PI]) - end; - _ -> {error, {no_pid, XPid}} - end. - -print_info(Device, Pid, {Module, Func}, Opt, Data) -> - case erlang:function_exported(Module, Func, 2) of - true -> - case catch apply(Module, Func, [Opt, Data]) of - Format when is_list(Format) -> - format_lib_supp:print_info(Device, 79, - add_pid_to_format(Pid, Format)), - ok; - Other -> {error, {'invalid format', Other}} - end; - _ -> - {error, {no_such_function, Module, Func}} - end. - -add_pid_to_format(Pid, [{header, H} | T]) -> - [{header, H}, {data, [{"Pid", Pid}]} | T]; -add_pid_to_format(Pid, List) -> - [{data, [{"Pid", Pid}]} | List]. - - diff --git a/lib/sasl/src/systools_make.erl b/lib/sasl/src/systools_make.erl index b1523dcbb7..391b1fb5cc 100644 --- a/lib/sasl/src/systools_make.erl +++ b/lib/sasl/src/systools_make.erl @@ -1152,10 +1152,10 @@ generate_script(Output, Release, Appls, Flags) -> }, ScriptFile = Output ++ ".script", - case file:open(ScriptFile, [write]) of + case file:open(ScriptFile, [write,{encoding,utf8}]) of {ok, Fd} -> - io:format(Fd, "%% script generated at ~w ~w\n~p.\n", - [date(), time(), Script]), + io:format(Fd, "%% ~s\n%% script generated at ~w ~w\n~tp.\n", + [epp:encoding_to_string(utf8), date(), time(), Script]), case file:close(Fd) of ok -> BootFile = Output ++ ".boot", diff --git a/lib/sasl/src/systools_relup.erl b/lib/sasl/src/systools_relup.erl index 706ae7d631..e836d57670 100644 --- a/lib/sasl/src/systools_relup.erl +++ b/lib/sasl/src/systools_relup.erl @@ -535,9 +535,9 @@ to_list(X) when is_list(X) -> X. write_relup_file(Relup, Opts) -> Filename = filename:join(filename:absname(get_opt(outdir,Opts)), "relup"), - case file:open(Filename, [write]) of + case file:open(Filename, [write,{encoding,utf8}]) of {ok, Fd} -> - io:format(Fd, "~p.~n", [Relup]), + io:format(Fd, "%% ~s~n~tp.~n", [epp:encoding_to_string(utf8),Relup]), case file:close(Fd) of ok -> ok; {error,Reason} -> diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl index 7093158502..50932e89e4 100644 --- a/lib/sasl/test/release_handler_SUITE.erl +++ b/lib/sasl/test/release_handler_SUITE.erl @@ -66,7 +66,7 @@ cases() -> supervisor_which_children_timeout, release_handler_which_releases, install_release_syntax_check, upgrade_supervisor, upgrade_supervisor_fail, otp_9864, - otp_10463_upgrade_script_regexp, no_dot_erlang]. + otp_10463_upgrade_script_regexp, no_dot_erlang, unicode_upgrade]. groups() -> [{release,[], @@ -1875,6 +1875,86 @@ no_dot_erlang(Conf) -> ok end. +%%%----------------------------------------------------------------- +%%% Test unicode handling. Make sure that release name, application +%%% description, and application environment variables may contain +%%% unicode characters. +unicode_upgrade(Conf) -> + %% Set some paths + DataDir = ?config(data_dir, Conf), + PrivDir = priv_dir(Conf), + Dir = filename:join(PrivDir,"unicode"), + LibDir0 = filename:join(DataDir, "unicode"), + LibDir = + case {file:native_name_encoding(),os:type()} of + {utf8,{Os,_}} when Os =/= win32 -> + LD = filename:join(DataDir,"unicode_αβ"), + file:make_symlink("unicode",LD), + LD; + _ -> + LibDir0 + end, + + %% Create the releases + RelName = "unicode_rel_αβ", + Rel1 = create_and_install_fake_first_release(Dir,{RelName,"1"}, + [{u,"1.0",LibDir}]), + Rel2 = create_fake_upgrade_release(Dir, + {RelName,"2"}, + [{u,"1.1",LibDir}], + {[Rel1],[Rel1],[LibDir]}), + Rel1Dir = filename:dirname(Rel1), + Rel2Dir = filename:dirname(Rel2), + + %% Start a slave node + {ok, Node} = t_start_node(unicode_upgrade, Rel1, + filename:join(Rel1Dir,"sys.config"), "+pc unicode"), + + %% Check + Dir1 = filename:join([LibDir, "u-1.0"]), + Dir1 = rpc:call(Node, code, lib_dir, [u]), + UBeam1 = filename:join([Dir1,"ebin","u.beam"]), + UBeam1 = rpc:call(Node,code,which,[u]), + {RelName,"1"} = rpc:call(Node,init,script_id,[]), + {Env,state} = rpc:call(Node,u,u,[]), + 'val_αβ' = proplists:get_value('key_αβ',Env), + [{RelName,"1",_,permanent}|_] = + rpc:call(Node,release_handler,which_releases,[]), + {ok,ReleasesDir} = rpc:call(Node,application,get_env,[sasl,releases_dir]), + {ok,[[{release,RelName,"1",_,_,permanent}|_]]} = + file:consult(filename:join(ReleasesDir,"RELEASES")), + + %% Install second release + {ok, RelVsn2} = + rpc:call(Node, release_handler, set_unpacked, + [Rel2++".rel", [{u,"1.1",LibDir}]]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "relup")]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "start.boot")]), + ok = rpc:call(Node, release_handler, install_file, + [RelVsn2, filename:join(Rel2Dir, "sys.config")]), + + {ok, _RelVsn1, []} = + rpc:call(Node, release_handler, install_release, [RelVsn2]), + + %% And check + Dir2 = filename:join([LibDir, "u-1.1"]), + Dir2 = rpc:call(Node, code, lib_dir, [u]), + UBeam2 = filename:join([Dir2,"ebin","u.beam"]), + {file,UBeam2} = rpc:call(Node,code,is_loaded,[u]), + {RelName,"1"} = rpc:call(Node,init,script_id,[]), + {Env,{state,'αβ'}} = rpc:call(Node,u,u,[]), + [{RelName,"2",_,current}|_] = + rpc:call(Node,release_handler,which_releases,[]), + {ok,ReleasesDir2} = rpc:call(Node,application,get_env,[sasl,releases_dir]), + {ok,<<"%% coding: utf-8\n[{release,\"unicode_rel_αβ\",\"2\""/utf8,_/binary>>}= + file:read_file(filename:join(ReleasesDir2,"RELEASES")), + ok. + +unicode_upgrade(cleanup,_Conf) -> + stop_node(node_name(unicode_upgrade)). + %%%================================================================= %%% Misceleaneous functions @@ -2002,6 +2082,8 @@ are_names_reg_gg(Node, Names, N) -> t_start_node(Name, Boot, SysConfig) -> + t_start_node(Name, Boot, SysConfig, ""). +t_start_node(Name, Boot, SysConfig, ArgStr) -> Args = case Boot of [] -> []; @@ -2010,8 +2092,9 @@ t_start_node(Name, Boot, SysConfig) -> case SysConfig of [] -> []; _ -> " -config " ++ SysConfig - end, - test_server:start_node(Name, slave, [{args, Args}]). + end ++ + " " ++ ArgStr, + test_server:start_node(Name, peer, [{args, Args}]). stop_node(Node) -> ?t:stop_node(Node). @@ -2460,7 +2543,9 @@ create_rel_file(RelFile,RelName,RelVsn,Erts,ExtraApps) -> %% Insert a term in a file, which can be read with file:consult/1. write_term_file(File,Term) -> - ok = file:write_file(File,io_lib:format("~p.~n",[Term])). + Str = io_lib:format("%% ~s~n~tp.~n",[epp:encoding_to_string(utf8),Term]), + Bin = unicode:characters_to_binary(Str), + ok = file:write_file(File,Bin). %% Check that global group info is correct - try again for a maximum of 5 sec @@ -2719,8 +2804,8 @@ cover_fun(Node,Func) -> %% and possibly other applications if they are listed in AppDirs = %% [{App,Vsn,LibDir}] create_and_install_fake_first_release(Dir,AppDirs) -> - %% Create the first release - {RelName,RelVsn} = init:script_id(), + create_and_install_fake_first_release(Dir,init:script_id(),AppDirs). +create_and_install_fake_first_release(Dir,{RelName,RelVsn},AppDirs) -> {Rel,_} = create_fake_release(Dir,RelName,RelVsn,AppDirs), ReleasesDir = filename:join(Dir, "releases"), RelDir = filename:dirname(Rel), @@ -2744,9 +2829,11 @@ create_and_install_fake_first_release(Dir,AppDirs) -> %% be upgraded to from the release created by %% create_and_install_fake_first_release/2. Unpack first by calls to %% release_handler:set_unpacked and release_handler:install_file. -create_fake_upgrade_release(Dir,RelVsn,AppDirs,{UpFrom,DownTo,ExtraLibs}) -> - %% Create a new release +create_fake_upgrade_release(Dir,RelVsn,AppDirs,UpgrInstr) when not is_tuple(RelVsn) -> {RelName,_} = init:script_id(), + create_fake_upgrade_release(Dir,{RelName,RelVsn},AppDirs,UpgrInstr); +create_fake_upgrade_release(Dir,{RelName,RelVsn},AppDirs,{UpFrom,DownTo,ExtraLibs}) -> + %% Create a new release {Rel,Paths} = create_fake_release(Dir,RelName,RelVsn,AppDirs), RelDir = filename:dirname(Rel), diff --git a/lib/sasl/test/release_handler_SUITE_data/Makefile.src b/lib/sasl/test/release_handler_SUITE_data/Makefile.src index b794aa0e6f..113d3e2290 100644 --- a/lib/sasl/test/release_handler_SUITE_data/Makefile.src +++ b/lib/sasl/test/release_handler_SUITE_data/Makefile.src @@ -76,7 +76,13 @@ SUP= \ release_handler_timeouts/dummy-0.1/ebin/dummy_sup.@EMULATOR@ \ release_handler_timeouts/dummy-0.1/ebin/dummy_sup_2.@EMULATOR@ -all: $(LIB) $(APP) $(OTP2740) $(C) $(SUP) +UNICODE= \ + unicode/u-1.0/ebin/u.@EMULATOR@ \ + unicode/u-1.0/ebin/u_sup.@EMULATOR@ \ + unicode/u-1.1/ebin/u.@EMULATOR@ \ + unicode/u-1.1/ebin/u_sup.@EMULATOR@ + +all: $(LIB) $(APP) $(OTP2740) $(C) $(SUP) $(UNICODE) lib/a-1.0/ebin/a.@EMULATOR@: lib/a-1.0/src/a.erl erlc $(EFLAGS) -olib/a-1.0/ebin lib/a-1.0/src/a.erl @@ -236,3 +242,13 @@ release_handler_timeouts/dummy-0.1/ebin/dummy_sup.@EMULATOR@: release_handler_ti erlc $(EFLAGS) -orelease_handler_timeouts/dummy-0.1/ebin release_handler_timeouts/dummy-0.1/src/dummy_sup.erl release_handler_timeouts/dummy-0.1/ebin/dummy_sup_2.@EMULATOR@: release_handler_timeouts/dummy-0.1/src/dummy_sup_2.erl erlc $(EFLAGS) -orelease_handler_timeouts/dummy-0.1/ebin release_handler_timeouts/dummy-0.1/src/dummy_sup_2.erl + +unicode/u-1.0/ebin/u.@EMULATOR@: unicode/u-1.0/src/u.erl + erlc $(EFLAGS) -ounicode/u-1.0/ebin unicode/u-1.0/src/u.erl +unicode/u-1.0/ebin/u_sup.@EMULATOR@: unicode/u-1.0/src/u_sup.erl + erlc $(EFLAGS) -ounicode/u-1.0/ebin unicode/u-1.0/src/u_sup.erl + +unicode/u-1.1/ebin/u.@EMULATOR@: unicode/u-1.1/src/u.erl + erlc $(EFLAGS) -ounicode/u-1.1/ebin unicode/u-1.1/src/u.erl +unicode/u-1.1/ebin/u_sup.@EMULATOR@: unicode/u-1.1/src/u_sup.erl + erlc $(EFLAGS) -ounicode/u-1.1/ebin unicode/u-1.1/src/u_sup.erl diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/ebin/u.app b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/ebin/u.app new file mode 100644 index 0000000000..fea4f9992e --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/ebin/u.app @@ -0,0 +1,8 @@ +{application, u, + [{description, "This app shall test unicode handling αβ"}, + {vsn, "1.0"}, + {modules, [u, u_sup]}, + {registered, [u_sup]}, + {applications, [kernel, stdlib]}, + {env, [{'key_αβ', 'val_αβ'}]}, + {mod, {u_sup, []}}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/src/u.erl b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/src/u.erl new file mode 100644 index 0000000000..45fe098c0e --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/src/u.erl @@ -0,0 +1,50 @@ +%% ``Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +%% $Id$ +%% +-module(u). + + +-behaviour(gen_server). + +-vsn(1). + +%% External exports +-export([start_link/0, u/0]). +%% Internal exports +-export([init/1, handle_call/3, handle_info/2, terminate/2]). + +start_link() -> gen_server:start_link({local, uu}, u, [], []). + +u() -> gen_server:call(uu, u). + +%%----------------------------------------------------------------- +%% Callback functions from gen_server +%%----------------------------------------------------------------- +init([]) -> + process_flag(trap_exit, true), + {ok, state}. + +handle_call(u, _From, State) -> + X = application:get_all_env(u), + {reply, {X,State}, State}. + +handle_info(_, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/src/u_sup.erl b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/src/u_sup.erl new file mode 100644 index 0000000000..b0d4a7b58f --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.0/src/u_sup.erl @@ -0,0 +1,38 @@ +%% ``Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +%% $Id$ +%% +-module(u_sup). + + +-behaviour(supervisor). + +%% External exports +-export([start/2]). + +%% Internal exports +-export([init/1]). + +start(_, _) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + SupFlags = {one_for_one, 4, 3600}, + Config = {u, + {u, start_link, []}, + permanent, 2000, worker, [u]}, + {ok, {SupFlags, [Config]}}. diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/ebin/u.app b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/ebin/u.app new file mode 100644 index 0000000000..8fcc3bba42 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/ebin/u.app @@ -0,0 +1,8 @@ +{application, u, + [{description, "This app shall test unicode handling αβ"}, + {vsn, "1.1"}, + {modules, [u, u_sup]}, + {registered, [u_sup]}, + {applications, [kernel, stdlib]}, + {env, [{'key_αβ', 'val_αβ'}]}, + {mod, {u_sup, []}}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/ebin/u.appup b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/ebin/u.appup new file mode 100644 index 0000000000..0344ce92ab --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/ebin/u.appup @@ -0,0 +1,3 @@ +{"1.1", + [{"1.0",[{update,u,{advanced,'αβ'}}]}], + [{"1.0",[{update,u,{advanced,'αβ'}}]}]}. diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/src/u.erl b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/src/u.erl new file mode 100644 index 0000000000..d2544d6fc1 --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/src/u.erl @@ -0,0 +1,55 @@ +%% ``Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +%% $Id$ +%% +-module(u). + + +-behaviour(gen_server). + +-vsn(1). + +%% External exports +-export([start_link/0, u/0]). +%% Internal exports +-export([init/1, handle_call/3, handle_info/2, terminate/2, code_change/3]). + +start_link() -> gen_server:start_link({local, uu}, u, [], []). + +u() -> gen_server:call(uu, u). + +%%----------------------------------------------------------------- +%% Callback functions from gen_server +%%----------------------------------------------------------------- +init([]) -> + process_flag(trap_exit, true), + {ok, {state,'αβ'}}. + +handle_call(u, _From, State) -> + X = application:get_all_env(u), + {reply, {X,State}, State}. + +handle_info(_, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change({down,_}, {State,_}, _Extra) -> + {ok, State}; +code_change(_, State, Extra) -> + {ok, {State, Extra}}. diff --git a/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/src/u_sup.erl b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/src/u_sup.erl new file mode 100644 index 0000000000..b0d4a7b58f --- /dev/null +++ b/lib/sasl/test/release_handler_SUITE_data/unicode/u-1.1/src/u_sup.erl @@ -0,0 +1,38 @@ +%% ``Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +%% $Id$ +%% +-module(u_sup). + + +-behaviour(supervisor). + +%% External exports +-export([start/2]). + +%% Internal exports +-export([init/1]). + +start(_, _) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + SupFlags = {one_for_one, 4, 3600}, + Config = {u, + {u, start_link, []}, + permanent, 2000, worker, [u]}, + {ok, {SupFlags, [Config]}}. diff --git a/lib/sasl/test/test_lib.hrl b/lib/sasl/test/test_lib.hrl index 9a54937f96..f5210d4f27 100644 --- a/lib/sasl/test/test_lib.hrl +++ b/lib/sasl/test/test_lib.hrl @@ -1,3 +1,3 @@ -define(ertsvsn,"4.4"). --define(kernelvsn,"5.0"). --define(stdlibvsn,"3.0"). +-define(kernelvsn,"5.3"). +-define(stdlibvsn,"3.4"). diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index f93753f1d2..5826d14a4a 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,22 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.5.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + All unknown options are sent to the transport handler + regardless of type.</p> + <p> + Own Id: OTP-14541 Aux Id: EIRERL-63 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.5</title> <section><title>Improvements and New Features</title> diff --git a/lib/ssh/src/ssh_io.erl b/lib/ssh/src/ssh_io.erl index 8ba759ad60..a7cd1daeec 100644 --- a/lib/ssh/src/ssh_io.erl +++ b/lib/ssh/src/ssh_io.erl @@ -31,8 +31,8 @@ read_line(Prompt, Opts) -> format("~s", [listify(Prompt)]), ?GET_INTERNAL_OPT(user_pid, Opts) ! {self(), question}, receive - Answer when is_list(Answer) -> - Answer + Answer when is_list(Answer) or is_binary(Answer) -> + unicode:characters_to_list(Answer) end. yes_no(Prompt, Opts) -> @@ -44,7 +44,7 @@ yes_no(Prompt, Opts) -> y -> yes; n -> no; - Answer when is_list(Answer) -> + Answer when is_list(Answer) or is_binary(Answer) -> case trim(Answer) of "y" -> yes; "n" -> no; @@ -60,7 +60,7 @@ read_password(Prompt, Opts) -> format("~s", [listify(Prompt)]), ?GET_INTERNAL_OPT(user_pid, Opts) ! {self(), user_password}, receive - Answer when is_list(Answer) -> + Answer when is_list(Answer) or is_binary(Answer) -> case trim(Answer) of "" -> read_password(Prompt, Opts); diff --git a/lib/ssh/src/ssh_options.erl b/lib/ssh/src/ssh_options.erl index 7eeed70739..b41ad8b33b 100644 --- a/lib/ssh/src/ssh_options.erl +++ b/lib/ssh/src/ssh_options.erl @@ -236,7 +236,10 @@ save({Key,Value}, Defs, OptMap) when is_map(OptMap) -> %% by the check fun will give an error exception: error:{check,{BadValue,Extra}} -> error({eoptions, {Key,BadValue}, Extra}) - end. + end; +save(Opt, _Defs, OptMap) when is_map(OptMap) -> + OptMap#{socket_options := [Opt | maps:get(socket_options,OptMap)]}. + %%%================================================================ %%% diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index c1558a19b1..9e1229dc85 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -1050,7 +1050,7 @@ attr_to_info(A) when is_record(A, ssh_xfer_attr) -> #file_info{ size = A#ssh_xfer_attr.size, type = A#ssh_xfer_attr.type, - access = read_write, %% FIXME: read/write/read_write/none + access = file_mode_to_owner_access(A#ssh_xfer_attr.permissions), atime = unix_to_datetime(A#ssh_xfer_attr.atime), mtime = unix_to_datetime(A#ssh_xfer_attr.mtime), ctime = unix_to_datetime(A#ssh_xfer_attr.createtime), @@ -1062,6 +1062,28 @@ attr_to_info(A) when is_record(A, ssh_xfer_attr) -> uid = A#ssh_xfer_attr.owner, gid = A#ssh_xfer_attr.group}. +file_mode_to_owner_access(FileMode) + when is_integer(FileMode) -> + %% The file mode contains the access permissions. + %% The read and write access permission of file owner + %% are located in 8th and 7th bit of file mode respectively. + + ReadPermission = ((FileMode bsr 8) band 1), + WritePermission = ((FileMode bsr 7) band 1), + case {ReadPermission, WritePermission} of + {1, 1} -> + read_write; + {1, 0} -> + read; + {0, 1} -> + write; + {0, 0} -> + none; + _ -> + undefined + end; +file_mode_to_owner_access(_) -> + undefined. unix_to_datetime(undefined) -> undefined; diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index 680a8ef52e..7aa3d8a00a 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -92,7 +92,7 @@ groups() -> {write_read_tests, [], [open_close_file, open_close_dir, read_file, read_dir, write_file, write_file_iolist, write_big_file, sftp_read_big_file, rename_file, mk_rm_dir, remove_file, links, - retrieve_attributes, set_attributes, async_read, + retrieve_attributes, set_attributes, file_owner_access, async_read, async_write, position, pos_read, pos_write, start_channel_sock ]} @@ -521,7 +521,36 @@ set_attributes(Config) when is_list(Config) -> ok = file:write_file(FileName, "hello again"). %%-------------------------------------------------------------------- +file_owner_access() -> + [{doc,"Test file user access validity"}]. +file_owner_access(Config) when is_list(Config) -> + case os:type() of + {win32, _} -> + {skip, "Not a relevant test on Windows"}; + _ -> + FileName = proplists:get_value(filename, Config), + {Sftp, _} = proplists:get_value(sftp, Config), + + {ok, #file_info{mode = InitialMode}} = ssh_sftp:read_file_info(Sftp, FileName), + + ok = ssh_sftp:write_file_info(Sftp, FileName, #file_info{mode=8#000}), + {ok, #file_info{access = none}} = ssh_sftp:read_file_info(Sftp, FileName), + + ok = ssh_sftp:write_file_info(Sftp, FileName, #file_info{mode=8#400}), + {ok, #file_info{access = read}} = ssh_sftp:read_file_info(Sftp, FileName), + + ok = ssh_sftp:write_file_info(Sftp, FileName, #file_info{mode=8#200}), + {ok, #file_info{access = write}} = ssh_sftp:read_file_info(Sftp, FileName), + ok = ssh_sftp:write_file_info(Sftp, FileName, #file_info{mode=8#600}), + {ok, #file_info{access = read_write}} = ssh_sftp:read_file_info(Sftp, FileName), + + ok = ssh_sftp:write_file_info(Sftp, FileName, #file_info{mode=InitialMode}), + + ok + end. + +%%-------------------------------------------------------------------- async_read() -> [{doc,"Test API aread/3"}]. async_read(Config) when is_list(Config) -> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 7208baca6e..006228f8e7 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,5 +1,5 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.5 +SSH_VSN = 4.5.1 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index e8cfbbe2e3..ff3e69bae5 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -48,7 +48,7 @@ select_sni_extension/1]). %% Alert and close handling --export([encode_alert/3,send_alert/2, close/5]). +-export([encode_alert/3,send_alert/2, close/5, protocol_name/0]). %% Data handling @@ -208,6 +208,9 @@ setopts(Transport, Socket, Other) -> getopts(Transport, Socket, Tag) -> dtls_socket:getopts(Transport, Socket, Tag). +protocol_name() -> + "DTLS". + %%==================================================================== %% tls_connection_sup API %%==================================================================== @@ -273,7 +276,9 @@ init({call, _} = Type, Event, #state{role = server, transport_cb = gen_udp} = St Result = ssl_connection:init(Type, Event, State#state{flight_state = {retransmit, ?INITIAL_RETRANSMIT_TIMEOUT}, protocol_specific = #{current_cookie_secret => dtls_v1:cookie_secret(), - previous_cookie_secret => <<>>}}, + previous_cookie_secret => <<>>, + ignored_alerts => 0, + max_ignored_alerts => 10}}, ?MODULE), erlang:send_after(dtls_v1:cookie_timeout(), self(), new_cookie_secret), Result; @@ -371,7 +376,7 @@ hello(internal, #server_hello{} = Hello, ssl_options = SslOptions} = State) -> case dtls_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, ReqVersion, hello, State); + handle_own_alert(Alert, ReqVersion, hello, State); {Version, NewId, ConnectionStates, ProtoExt, Protocol} -> ssl_connection:handle_session(Hello, Version, NewId, ConnectionStates, ProtoExt, Protocol, State) @@ -543,7 +548,7 @@ handle_call(Event, From, StateName, State) -> handle_common_event(internal, #alert{} = Alert, StateName, #state{negotiated_version = Version} = State) -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State); + handle_own_alert(Alert, Version, StateName, State); %%% DTLS record protocol level handshake messages handle_common_event(internal, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, @@ -562,7 +567,7 @@ handle_common_event(internal, #ssl_tls{type = ?HANDSHAKE, State#state{unprocessed_handshake_events = unprocessed_events(Events)}, Events} end catch throw:#alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State0) + handle_own_alert(Alert, Version, StateName, State0) end; %%% DTLS record protocol level application data messages handle_common_event(internal, #ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, State) -> @@ -577,7 +582,7 @@ handle_common_event(internal, #ssl_tls{type = ?ALERT, fragment = EncAlerts}, Sta Alerts = [_|_] -> handle_alerts(Alerts, {next_state, StateName, State}); #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, Version, StateName, State) + handle_own_alert(Alert, Version, StateName, State) end; %% Ignore unknown TLS record level protocol messages handle_common_event(internal, #ssl_tls{type = _Unknown}, StateName, State) -> @@ -629,7 +634,7 @@ handle_client_hello(#client_hello{client_version = ClientVersion} = Hello, case dtls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb, ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of #alert{} = Alert -> - ssl_connection:handle_own_alert(Alert, ClientVersion, hello, State0); + handle_own_alert(Alert, ClientVersion, hello, State0); {Version, {Type, Session}, ConnectionStates, Protocol0, ServerHelloExt, HashSign} -> Protocol = case Protocol0 of @@ -964,3 +969,54 @@ unprocessed_events(Events) -> %% process more TLS-records received on the socket. erlang:length(Events)-1. +handle_own_alert(Alert, Version, StateName, #state{transport_cb = gen_udp, + role = Role, + ssl_options = Options} = State0) -> + case ignore_alert(Alert, State0) of + {true, State} -> + log_ignore_alert(Options#ssl_options.log_alert, StateName, Alert, Role), + {next_state, StateName, State}; + {false, State} -> + ssl_connection:handle_own_alert(Alert, Version, StateName, State) + end; +handle_own_alert(Alert, Version, StateName, State) -> + ssl_connection:handle_own_alert(Alert, Version, StateName, State). + + +ignore_alert(#alert{level = ?FATAL}, #state{protocol_specific = #{ignored_alerts := N, + max_ignored_alerts := N}} = State) -> + {false, State}; +ignore_alert(#alert{level = ?FATAL} = Alert, + #state{protocol_specific = #{ignored_alerts := N} = PS} = State) -> + case is_ignore_alert(Alert) of + true -> + {true, State#state{protocol_specific = PS#{ignored_alerts => N+1}}}; + false -> + {false, State} + end; +ignore_alert(_, State) -> + {false, State}. + +%% RFC 6347 4.1.2.7. Handling Invalid Records +%% recommends to silently ignore invalid DTLS records when +%% upd is the transport. Note we do not support compression so no need +%% include ?DECOMPRESSION_FAILURE +is_ignore_alert(#alert{description = ?BAD_RECORD_MAC}) -> + true; +is_ignore_alert(#alert{description = ?RECORD_OVERFLOW}) -> + true; +is_ignore_alert(#alert{description = ?DECODE_ERROR}) -> + true; +is_ignore_alert(#alert{description = ?DECRYPT_ERROR}) -> + true; +is_ignore_alert(#alert{description = ?ILLEGAL_PARAMETER}) -> + true; +is_ignore_alert(_) -> + false. + +log_ignore_alert(true, StateName, Alert, Role) -> + Txt = ssl_alert:alert_txt(Alert), + error_logger:format("DTLS over UDP ~p: In state ~p ignored to send ALERT ~s as DoS-attack mitigation \n", + [Role, StateName, Txt]); +log_ignore_alert(false, _, _,_) -> + ok. diff --git a/lib/ssl/src/dtls_socket.erl b/lib/ssl/src/dtls_socket.erl index fbbd479428..5f854fbb4b 100644 --- a/lib/ssl/src/dtls_socket.erl +++ b/lib/ssl/src/dtls_socket.erl @@ -137,7 +137,7 @@ internal_inet_values() -> [{active, false}, {mode,binary}]. default_inet_values() -> - [{active, true}, {mode, list}]. + [{active, true}, {mode, list}, {packet, 0}, {packet_size, 0}]. default_cb_info() -> {gen_udp, udp, udp_closed, udp_error}. @@ -149,8 +149,12 @@ get_emulated_opts(EmOpts, EmOptNames) -> emulated_socket_options(InetValues, #socket_options{ mode = Mode, + packet = Packet, + packet_size = PacketSize, active = Active}) -> #socket_options{ mode = proplists:get_value(mode, InetValues, Mode), + packet = proplists:get_value(packet, InetValues, Packet), + packet_size = proplists:get_value(packet_size, InetValues, PacketSize), active = proplists:get_value(active, InetValues, Active) }. diff --git a/lib/ssl/src/dtls_udp_listener.erl b/lib/ssl/src/dtls_udp_listener.erl index c789a32087..c9e04767aa 100644 --- a/lib/ssl/src/dtls_udp_listener.erl +++ b/lib/ssl/src/dtls_udp_listener.erl @@ -35,7 +35,7 @@ -record(state, {port, - listner, + listener, dtls_options, emulated_options, dtls_msq_queues = kv_new(), @@ -81,7 +81,7 @@ init([Port, EmOpts, InetOptions, DTLSOptions]) -> first = true, dtls_options = DTLSOptions, emulated_options = EmOpts, - listner = Socket, + listener = Socket, close = false}} catch _:_ -> {error, closed} @@ -91,7 +91,7 @@ handle_call({accept, _}, _, #state{close = true} = State) -> handle_call({accept, Accepter}, From, #state{first = true, accepters = Accepters, - listner = Socket} = State0) -> + listener = Socket} = State0) -> next_datagram(Socket), State = State0#state{first = false, accepters = queue:in({Accepter, From}, Accepters)}, @@ -100,7 +100,7 @@ handle_call({accept, Accepter}, From, #state{first = true, handle_call({accept, Accepter}, From, #state{accepters = Accepters} = State0) -> State = State0#state{accepters = queue:in({Accepter, From}, Accepters)}, {noreply, State}; -handle_call(sockname, _, #state{listner = Socket} = State) -> +handle_call(sockname, _, #state{listener = Socket} = State) -> Reply = inet:sockname(Socket), {reply, Reply, State}; handle_call(close, _, #state{dtls_processes = Processes, @@ -114,7 +114,7 @@ handle_call(close, _, #state{dtls_processes = Processes, end, queue:to_list(Accepters)), {reply, ok, State#state{close = true, accepters = queue:new()}} end; -handle_call({get_sock_opts, {SocketOptNames, EmOptNames}}, _, #state{listner = Socket, +handle_call({get_sock_opts, {SocketOptNames, EmOptNames}}, _, #state{listener = Socket, emulated_options = EmOpts} = State) -> case get_socket_opts(Socket, SocketOptNames) of {ok, Opts} -> @@ -125,7 +125,7 @@ handle_call({get_sock_opts, {SocketOptNames, EmOptNames}}, _, #state{listner = S handle_call(get_all_opts, _, #state{dtls_options = DTLSOptions, emulated_options = EmOpts} = State) -> {reply, {ok, EmOpts, DTLSOptions}, State}; -handle_call({set_sock_opts, {SocketOpts, NewEmOpts}}, _, #state{listner = Socket, emulated_options = EmOpts0} = State) -> +handle_call({set_sock_opts, {SocketOpts, NewEmOpts}}, _, #state{listener = Socket, emulated_options = EmOpts0} = State) -> set_socket_opts(Socket, SocketOpts), EmOpts = do_set_emulated_opts(NewEmOpts, EmOpts0), {reply, ok, State#state{emulated_options = EmOpts}}. @@ -134,7 +134,7 @@ handle_cast({active_once, Client, Pid}, State0) -> State = handle_active_once(Client, Pid, State0), {noreply, State}. -handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listner = Socket} = State0) -> +handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listener = Socket} = State0) -> State = handle_datagram({IP, InPortNo}, Msg, State0), next_datagram(Socket), {noreply, State}; @@ -142,11 +142,11 @@ handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listner = Socket} = Sta %% UDP socket does not have a connection and should not receive an econnreset %% This does however happens on on some windows versions. Just ignoring it %% appears to make things work as expected! -handle_info({udp_error, Socket, econnreset = Error}, #state{listner = Socket} = State) -> +handle_info({udp_error, Socket, econnreset = Error}, #state{listener = Socket} = State) -> Report = io_lib:format("Ignore SSL UDP Listener: Socket error: ~p ~n", [Error]), error_logger:info_report(Report), {noreply, State}; -handle_info({udp_error, Socket, Error}, #state{listner = Socket} = State) -> +handle_info({udp_error, Socket, Error}, #state{listener = Socket} = State) -> Report = io_lib:format("SSL UDP Listener shutdown: Socket error: ~p ~n", [Error]), error_logger:info_report(Report), {noreply, State#state{close=true}}; @@ -225,10 +225,10 @@ setup_new_connection(User, From, Client, Msg, #state{dtls_processes = Processes, dtls_msq_queues = MsgQueues, dtls_options = DTLSOpts, port = Port, - listner = Socket, + listener = Socket, emulated_options = EmOpts} = State) -> ConnArgs = [server, "localhost", Port, {self(), {Client, Socket}}, - {DTLSOpts, EmOpts, udp_listner}, User, dtls_socket:default_cb_info()], + {DTLSOpts, EmOpts, udp_listener}, User, dtls_socket:default_cb_info()], case dtls_connection_sup:start_child(ConnArgs) of {ok, Pid} -> erlang:monitor(process, Pid), diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 75eb308ba5..4e592c02ec 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -569,7 +569,7 @@ renegotiate(#sslsocket{pid = {Listen,_}}) when is_port(Listen) -> %%-------------------------------------------------------------------- -spec prf(#sslsocket{}, binary() | 'master_secret', binary(), - binary() | prf_random(), non_neg_integer()) -> + [binary() | prf_random()], non_neg_integer()) -> {ok, binary()} | {error, reason()}. %% %% Description: use a ssl sessions TLS PRF to generate key material @@ -713,6 +713,13 @@ handle_options(Opts0, Role, Host) -> Protocol = handle_option(protocol, Opts, tls), + case Versions of + [{3, 0}] -> + reject_alpn_next_prot_options(Opts); + _ -> + ok + end, + SSLOptions = #ssl_options{ versions = Versions, verify = validate_option(verify, Verify), @@ -809,7 +816,7 @@ handle_options(Opts0, Role, Host) -> ConnetionCb = connection_cb(Opts), {ok, #config{ssl = SSLOptions, emulated = Emulated, inet_ssl = Sock, - inet_user = SockOpts, transport_info = CbInfo, connection_cb = ConnetionCb + inet_user = Sock, transport_info = CbInfo, connection_cb = ConnetionCb }}. @@ -956,55 +963,32 @@ validate_option(hibernate_after, Value) when is_integer(Value), Value >= 0 -> validate_option(erl_dist,Value) when is_boolean(Value) -> Value; -validate_option(Opt, Value) - when Opt =:= alpn_advertised_protocols orelse Opt =:= alpn_preferred_protocols, - is_list(Value) -> - case tls_record:highest_protocol_version([]) of - {3,0} -> - throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); - _ -> - validate_binary_list(Opt, Value), - Value - end; +validate_option(Opt, Value) when Opt =:= alpn_advertised_protocols orelse Opt =:= alpn_preferred_protocols, + is_list(Value) -> + validate_binary_list(Opt, Value), + Value; validate_option(Opt, Value) when Opt =:= alpn_advertised_protocols orelse Opt =:= alpn_preferred_protocols, Value =:= undefined -> undefined; -validate_option(client_preferred_next_protocols = Opt, {Precedence, PreferredProtocols} = Value) +validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols}) when is_list(PreferredProtocols) -> - case tls_record:highest_protocol_version([]) of - {3,0} -> - throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); - _ -> - validate_binary_list(client_preferred_next_protocols, PreferredProtocols), - validate_npn_ordering(Precedence), - {Precedence, PreferredProtocols, ?NO_PROTOCOL} - end; -validate_option(client_preferred_next_protocols = Opt, {Precedence, PreferredProtocols, Default} = Value) - when is_list(PreferredProtocols), is_binary(Default), - byte_size(Default) > 0, byte_size(Default) < 256 -> - case tls_record:highest_protocol_version([]) of - {3,0} -> - throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); - _ -> - validate_binary_list(client_preferred_next_protocols, PreferredProtocols), - validate_npn_ordering(Precedence), - Value - end; - + validate_binary_list(client_preferred_next_protocols, PreferredProtocols), + validate_npn_ordering(Precedence), + {Precedence, PreferredProtocols, ?NO_PROTOCOL}; +validate_option(client_preferred_next_protocols, {Precedence, PreferredProtocols, Default} = Value) + when is_list(PreferredProtocols), is_binary(Default), + byte_size(Default) > 0, byte_size(Default) < 256 -> + validate_binary_list(client_preferred_next_protocols, PreferredProtocols), + validate_npn_ordering(Precedence), + Value; validate_option(client_preferred_next_protocols, undefined) -> undefined; validate_option(log_alert, Value) when is_boolean(Value) -> Value; -validate_option(next_protocols_advertised = Opt, Value) when is_list(Value) -> - case tls_record:highest_protocol_version([]) of - {3,0} -> - throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); - _ -> - validate_binary_list(next_protocols_advertised, Value), - Value - end; - +validate_option(next_protocols_advertised, Value) when is_list(Value) -> + validate_binary_list(next_protocols_advertised, Value), + Value; validate_option(next_protocols_advertised, undefined) -> undefined; validate_option(server_name_indication = Opt, Value) when is_list(Value) -> @@ -1483,3 +1467,22 @@ server_name_indication_default(Host) when is_list(Host) -> Host; server_name_indication_default(_) -> undefined. + + +reject_alpn_next_prot_options(Opts) -> + AlpnNextOpts = [alpn_advertised_protocols, + alpn_preferred_protocols, + next_protocols_advertised, + next_protocol_selector, + client_preferred_next_protocols], + reject_alpn_next_prot_options(AlpnNextOpts, Opts). + +reject_alpn_next_prot_options([], _) -> + ok; +reject_alpn_next_prot_options([Opt| AlpnNextOpts], Opts) -> + case lists:keyfind(Opt, 1, Opts) of + {Opt, Value} -> + throw({error, {options, {not_supported_in_sslv3, {Opt, Value}}}}); + false -> + reject_alpn_next_prot_options(AlpnNextOpts, Opts) + end. diff --git a/lib/ssl/src/ssl_alert.erl b/lib/ssl/src/ssl_alert.erl index 696a55e4b9..db415a3666 100644 --- a/lib/ssl/src/ssl_alert.erl +++ b/lib/ssl/src/ssl_alert.erl @@ -32,7 +32,7 @@ -include("ssl_record.hrl"). -include("ssl_internal.hrl"). --export([decode/1, alert_txt/1, reason_code/2]). +-export([decode/1, own_alert_txt/1, alert_txt/1, reason_code/2]). %%==================================================================== %% Internal application API @@ -57,16 +57,32 @@ decode(Bin) -> reason_code(#alert{description = ?CLOSE_NOTIFY}, _) -> closed; reason_code(#alert{description = Description}, _) -> - {tls_alert, description_txt(Description)}. + {tls_alert, string:to_lower(description_txt(Description))}. + +%%-------------------------------------------------------------------- +-spec own_alert_txt(#alert{}) -> string(). +%% +%% Description: Returns the error string for given alert generated +%% by the erlang implementation. +%%-------------------------------------------------------------------- +own_alert_txt(#alert{level = Level, description = Description, where = {Mod,Line}, reason = undefined, role = Role}) -> + "at " ++ Mod ++ ":" ++ integer_to_list(Line) ++ " generated " ++ string:to_upper(atom_to_list(Role)) ++ " ALERT: " ++ + level_txt(Level) ++ description_txt(Description); +own_alert_txt(#alert{reason = Reason} = Alert) -> + BaseTxt = own_alert_txt(Alert#alert{reason = undefined}), + FormatDepth = 9, % Some limit on printed representation of an error + ReasonTxt = lists:flatten(io_lib:format("~P", [Reason, FormatDepth])), + BaseTxt ++ " - " ++ ReasonTxt. %%-------------------------------------------------------------------- -spec alert_txt(#alert{}) -> string(). %% -%% Description: Returns the error string for given alert. +%% Description: Returns the error string for given alert received from +%% the peer. %%-------------------------------------------------------------------- -alert_txt(#alert{level = Level, description = Description, where = {Mod,Line}, reason = undefined}) -> - Mod ++ ":" ++ integer_to_list(Line) ++ ":" ++ - level_txt(Level) ++" "++ description_txt(Description); +alert_txt(#alert{level = Level, description = Description, reason = undefined, role = Role}) -> + "received " ++ string:to_upper(atom_to_list(Role)) ++ " ALERT: " ++ + level_txt(Level) ++ description_txt(Description); alert_txt(#alert{reason = Reason} = Alert) -> BaseTxt = alert_txt(Alert#alert{reason = undefined}), FormatDepth = 9, % Some limit on printed representation of an error @@ -93,73 +109,73 @@ decode(<<>>, Acc, _) -> lists:reverse(Acc, []). level_txt(?WARNING) -> - "Warning:"; + "Warning - "; level_txt(?FATAL) -> - "Fatal error:". + "Fatal - ". description_txt(?CLOSE_NOTIFY) -> - "close notify"; + "Close Notify"; description_txt(?UNEXPECTED_MESSAGE) -> - "unexpected message"; + "Unexpected Message"; description_txt(?BAD_RECORD_MAC) -> - "bad record mac"; -description_txt(?DECRYPTION_FAILED) -> - "decryption failed"; + "Bad Record MAC"; +description_txt(?DECRYPTION_FAILED_RESERVED) -> + "Decryption Failed Reserved"; description_txt(?RECORD_OVERFLOW) -> - "record overflow"; + "Record Overflow"; description_txt(?DECOMPRESSION_FAILURE) -> - "decompression failure"; + "Decompression Failure"; description_txt(?HANDSHAKE_FAILURE) -> - "handshake failure"; + "Handshake Failure"; description_txt(?NO_CERTIFICATE_RESERVED) -> - "No certificate reserved"; + "No Certificate Reserved"; description_txt(?BAD_CERTIFICATE) -> - "bad certificate"; + "Bad Certificate"; description_txt(?UNSUPPORTED_CERTIFICATE) -> - "unsupported certificate"; + "Unsupported Certificate"; description_txt(?CERTIFICATE_REVOKED) -> - "certificate revoked"; + "Certificate Revoked"; description_txt(?CERTIFICATE_EXPIRED) -> - "certificate expired"; + "Certificate Expired"; description_txt(?CERTIFICATE_UNKNOWN) -> - "certificate unknown"; + "Certificate Unknown"; description_txt(?ILLEGAL_PARAMETER) -> - "illegal parameter"; + "Illegal Parameter"; description_txt(?UNKNOWN_CA) -> - "unknown ca"; + "Unknown CA"; description_txt(?ACCESS_DENIED) -> - "access denied"; + "Access Denied"; description_txt(?DECODE_ERROR) -> - "decode error"; + "Decode Error"; description_txt(?DECRYPT_ERROR) -> - "decrypt error"; + "Decrypt Error"; description_txt(?EXPORT_RESTRICTION) -> - "export restriction"; + "Export Restriction"; description_txt(?PROTOCOL_VERSION) -> - "protocol version"; + "Protocol Version"; description_txt(?INSUFFICIENT_SECURITY) -> - "insufficient security"; + "Insufficient Security"; description_txt(?INTERNAL_ERROR) -> - "internal error"; + "Internal Error"; description_txt(?USER_CANCELED) -> - "user canceled"; + "User Canceled"; description_txt(?NO_RENEGOTIATION) -> - "no renegotiation"; + "No Renegotiation"; description_txt(?UNSUPPORTED_EXTENSION) -> - "unsupported extension"; + "Unsupported Extension"; description_txt(?CERTIFICATE_UNOBTAINABLE) -> - "certificate unobtainable"; + "Certificate Unobtainable"; description_txt(?UNRECOGNISED_NAME) -> - "unrecognised name"; + "Unrecognised Name"; description_txt(?BAD_CERTIFICATE_STATUS_RESPONSE) -> - "bad certificate status response"; + "Bad Certificate Status Response"; description_txt(?BAD_CERTIFICATE_HASH_VALUE) -> - "bad certificate hash value"; + "Bad Certificate Hash Value"; description_txt(?UNKNOWN_PSK_IDENTITY) -> - "unknown psk identity"; + "Unknown Psk Identity"; description_txt(?INAPPROPRIATE_FALLBACK) -> - "inappropriate fallback"; + "Inappropriate Fallback"; description_txt(?NO_APPLICATION_PROTOCOL) -> - "no application protocol"; + "No application protocol"; description_txt(Enum) -> lists:flatten(io_lib:format("unsupported/unknown alert: ~p", [Enum])). diff --git a/lib/ssl/src/ssl_alert.hrl b/lib/ssl/src/ssl_alert.hrl index f3743ba0f0..35670edea5 100644 --- a/lib/ssl/src/ssl_alert.hrl +++ b/lib/ssl/src/ssl_alert.hrl @@ -40,7 +40,7 @@ %% close_notify(0), %% unexpected_message(10), %% bad_record_mac(20), -%% decryption_failed(21), +%% decryption_failed_reserved(21), %% record_overflow(22), %% decompression_failure(30), %% handshake_failure(40), @@ -78,7 +78,7 @@ -define(CLOSE_NOTIFY, 0). -define(UNEXPECTED_MESSAGE, 10). -define(BAD_RECORD_MAC, 20). --define(DECRYPTION_FAILED, 21). +-define(DECRYPTION_FAILED_RESERVED, 21). -define(RECORD_OVERFLOW, 22). -define(DECOMPRESSION_FAILURE, 30). -define(HANDSHAKE_FAILURE, 40). @@ -118,6 +118,7 @@ level, description, where = {?FILE, ?LINE}, + role, reason }). -endif. % -ifdef(ssl_alert). diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index bd60197c88..b6cd22dd13 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -335,7 +335,9 @@ all_suites(Version) -> anonymous_suites({3, N}) -> anonymous_suites(N); - +anonymous_suites({254, _} = Version) -> + anonymous_suites(dtls_v1:corresponding_tls_version(Version)) + -- [?TLS_DH_anon_WITH_RC4_128_MD5]; anonymous_suites(N) when N >= 3 -> [?TLS_DH_anon_WITH_AES_128_GCM_SHA256, @@ -373,30 +375,38 @@ psk_suites({3, N}) -> psk_suites(N) when N >= 3 -> [ + ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384, ?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, ?TLS_PSK_WITH_AES_256_GCM_SHA384, + ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384, ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384, ?TLS_PSK_WITH_AES_256_CBC_SHA384, + ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256, ?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256, ?TLS_PSK_WITH_AES_128_GCM_SHA256, + ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256, ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256, ?TLS_PSK_WITH_AES_128_CBC_SHA256 ] ++ psk_suites(0); psk_suites(_) -> - [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, + [?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, + ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA, ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA, ?TLS_PSK_WITH_AES_256_CBC_SHA, + ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, ?TLS_DHE_PSK_WITH_AES_128_CBC_SHA, ?TLS_RSA_PSK_WITH_AES_128_CBC_SHA, ?TLS_PSK_WITH_AES_128_CBC_SHA, + ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, ?TLS_PSK_WITH_3DES_EDE_CBC_SHA, + ?TLS_ECDHE_PSK_WITH_RC4_128_SHA, ?TLS_DHE_PSK_WITH_RC4_128_SHA, ?TLS_RSA_PSK_WITH_RC4_128_SHA, ?TLS_PSK_WITH_RC4_128_SHA]. @@ -563,6 +573,15 @@ suite_definition(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA) -> suite_definition(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA) -> {rsa_psk, aes_256_cbc, sha, default_prf}; +%%% PSK NULL Cipher Suites RFC 4785 + +suite_definition(?TLS_PSK_WITH_NULL_SHA) -> + {psk, null, sha, default_prf}; +suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA) -> + {dhe_psk, null, sha, default_prf}; +suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA) -> + {rsa_psk, null, sha, default_prf}; + %%% TLS 1.2 PSK Cipher Suites RFC 5487 suite_definition(?TLS_PSK_WITH_AES_128_GCM_SHA256) -> @@ -604,6 +623,36 @@ suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA256) -> suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA384) -> {rsa_psk, null, sha384, default_prf}; +%%% ECDHE PSK Cipher Suites RFC 5489 + +suite_definition(?TLS_ECDHE_PSK_WITH_RC4_128_SHA) -> + {ecdhe_psk, rc4_128, sha, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA) -> + {ecdhe_psk, '3des_ede_cbc', sha, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA) -> + {ecdhe_psk, aes_128_cbc, sha, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA) -> + {ecdhe_psk, aes_256_cbc, sha, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256) -> + {ecdhe_psk, aes_128_cbc, sha256, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384) -> + {ecdhe_psk, aes_256_cbc, sha384, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA256) -> + {ecdhe_psk, null, sha256, default_prf}; +suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA384) -> + {ecdhe_psk, null, sha384, default_prf}; + +%%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 + +suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256) -> + {ecdhe_psk, aes_128_gcm, null, sha256}; +suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) -> + {ecdhe_psk, aes_256_gcm, null, sha384}; +%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) -> +%% {ecdhe_psk, aes_128_ccm, null, sha256}; +%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) -> +%% {ecdhe_psk, aes_256_ccm, null, sha256}; + %%% SRP Cipher Suites RFC 5054 suite_definition(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) -> @@ -865,6 +914,15 @@ suite({rsa_psk, aes_128_cbc,sha}) -> suite({rsa_psk, aes_256_cbc,sha}) -> ?TLS_RSA_PSK_WITH_AES_256_CBC_SHA; +%%% PSK NULL Cipher Suites RFC 4785 + +suite({psk, null, sha}) -> + ?TLS_PSK_WITH_NULL_SHA; +suite({dhe_psk, null, sha}) -> + ?TLS_DHE_PSK_WITH_NULL_SHA; +suite({rsa_psk, null, sha}) -> + ?TLS_RSA_PSK_WITH_NULL_SHA; + %%% TLS 1.2 PSK Cipher Suites RFC 5487 suite({psk, aes_128_gcm, null, sha256}) -> @@ -906,6 +964,36 @@ suite({rsa_psk, null, sha256}) -> suite({rsa_psk, null, sha384}) -> ?TLS_RSA_PSK_WITH_NULL_SHA384; +%%% ECDHE PSK Cipher Suites RFC 5489 + +suite({ecdhe_psk, rc4_128,sha}) -> + ?TLS_ECDHE_PSK_WITH_RC4_128_SHA; +suite({ecdhe_psk, '3des_ede_cbc',sha}) -> + ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA; +suite({ecdhe_psk, aes_128_cbc,sha}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA; +suite({ecdhe_psk, aes_256_cbc,sha}) -> + ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA; +suite({ecdhe_psk, aes_128_cbc, sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256; +suite({ecdhe_psk, aes_256_cbc, sha384}) -> + ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384; +suite({ecdhe_psk, null, sha256}) -> + ?TLS_ECDHE_PSK_WITH_NULL_SHA256; +suite({ecdhe_psk, null, sha384}) -> + ?TLS_ECDHE_PSK_WITH_NULL_SHA384; + +%%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 + +suite({ecdhe_psk, aes_128_gcm, null, sha256}) -> + ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256; +suite({ecdhe_psk, aes_256_gcm, null, sha384}) -> + ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384; +%% suite({ecdhe_psk, aes_128_ccm, null, sha256}) -> +%% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256; +%% suite({ecdhe_psk, aes_256_ccm, null, sha256}) -> +%% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256; + %%% SRP Cipher Suites RFC 5054 suite({srp_anon, '3des_ede_cbc', sha}) -> @@ -1465,7 +1553,8 @@ is_acceptable_keyexchange(dhe_dss, Algos) -> is_acceptable_keyexchange(dhe_rsa, Algos) -> proplists:get_bool(dh, Algos) andalso proplists:get_bool(rsa, Algos); -is_acceptable_keyexchange(ecdh_anon, Algos) -> +is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_anon; + KeyExchange == ecdhe_psk -> proplists:get_bool(ecdh, Algos); is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_ecdsa; KeyExchange == ecdhe_ecdsa -> diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 8e8f3d9c67..e5462d8402 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -399,6 +399,17 @@ %% TLS_RSA_PSK_WITH_AES_256_CBC_SHA = { 0x00, 0x95 }; -define(TLS_RSA_PSK_WITH_AES_256_CBC_SHA, <<?BYTE(16#00), ?BYTE(16#95)>>). +%%% PSK NULL Cipher Suites RFC 4785 + +%% TLS_PSK_WITH_NULL_SHA = { 0x00, 0x2C }; +-define(TLS_PSK_WITH_NULL_SHA, <<?BYTE(16#00), ?BYTE(16#2C)>>). + +%% TLS_DHE_PSK_WITH_NULL_SHA = { 0x00, 0x2D }; +-define(TLS_DHE_PSK_WITH_NULL_SHA, <<?BYTE(16#00), ?BYTE(16#2D)>>). + +%% TLS_RSA_PSK_WITH_NULL_SHA = { 0x00, 0x2E }; +-define(TLS_RSA_PSK_WITH_NULL_SHA, <<?BYTE(16#00), ?BYTE(16#2E)>>). + %%% TLS 1.2 PSK Cipher Suites RFC 5487 %% TLS_PSK_WITH_AES_128_GCM_SHA256 = {0x00,0xA8}; @@ -455,6 +466,46 @@ %% TLS_RSA_PSK_WITH_NULL_SHA384 = {0x00,0xB9}; -define(TLS_RSA_PSK_WITH_NULL_SHA384, <<?BYTE(16#00), ?BYTE(16#B9)>>). +%%% ECDHE PSK Cipher Suites RFC 5489 + +%% TLS_ECDHE_PSK_WITH_RC4_128_SHA = {0xC0,0x33}; +-define(TLS_ECDHE_PSK_WITH_RC4_128_SHA, <<?BYTE(16#C0), ?BYTE(16#33)>>). + +%% TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = {0xC0,0x34}; +-define(TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#34)>>). + +%% TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = {0xC0,0x35}; +-define(TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#35)>>). + +%% TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = {0xC0,0x36}; +-define(TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#36)>>). + +%% TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = {0xC0,0x37}; +-define(TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, <<?BYTE(16#C0), ?BYTE(16#37)>>). + +%% TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = {0xC0,0x38}; +-define(TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, <<?BYTE(16#C0), ?BYTE(16#38)>>). + +%% TLS_ECDHE_PSK_WITH_NULL_SHA256 = {0xC0,0x3A}; +-define(TLS_ECDHE_PSK_WITH_NULL_SHA256, <<?BYTE(16#C0), ?BYTE(16#3A)>>). + +%% TLS_ECDHE_PSK_WITH_NULL_SHA384 = {0xC0,0x3B}; +-define(TLS_ECDHE_PSK_WITH_NULL_SHA384, <<?BYTE(16#C0), ?BYTE(16#3B)>>). + +%%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05 + +%% TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256 = {0xTBD; 0xTBD} {0xD0,0x01}; +-define(TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, <<?BYTE(16#D0), ?BYTE(16#01)>>). + +%% TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384 = {0xTBD; 0xTBD} {0xD0,0x02}; +-define(TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384, <<?BYTE(16#D0), ?BYTE(16#02)>>). + +%% TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256 = {0xTBD; 0xTBD} {0xD0,0x03}; +-define(TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, <<?BYTE(16#D0), ?BYTE(16#03)>>). + +%% TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256 = {0xTBD; 0xTBD} {0xD0,0x05}; +-define(TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256, <<?BYTE(16#D0), ?BYTE(16#05)>>). + %%% SRP Cipher Suites RFC 5054 %% TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = { 0xC0,0x1A }; diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index fb87662c7b..5d48325719 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -146,8 +146,8 @@ socket_control(Connection, Socket, Pid, Transport) -> -spec socket_control(tls_connection | dtls_connection, port(), pid(), atom(), pid()| undefined) -> {ok, #sslsocket{}} | {error, reason()}. %%-------------------------------------------------------------------- -socket_control(Connection, Socket, Pid, Transport, udp_listner) -> - %% dtls listner process must have the socket control +socket_control(Connection, Socket, Pid, Transport, udp_listener) -> + %% dtls listener process must have the socket control {ok, Connection:socket(Pid, Transport, Socket, Connection, undefined)}; socket_control(tls_connection = Connection, Socket, Pid, Transport, ListenTracker) -> @@ -264,7 +264,7 @@ renegotiation(ConnectionPid) -> %%-------------------------------------------------------------------- -spec prf(pid(), binary() | 'master_secret', binary(), - binary() | ssl:prf_random(), non_neg_integer()) -> + [binary() | ssl:prf_random()], non_neg_integer()) -> {ok, binary()} | {error, reason()} | {'EXIT', term()}. %% %% Description: use a ssl sessions TLS PRF to generate key material @@ -517,7 +517,7 @@ certify(internal, #server_key_exchange{exchange_keys = Keys}, when Alg == dhe_dss; Alg == dhe_rsa; Alg == ecdhe_rsa; Alg == ecdhe_ecdsa; Alg == dh_anon; Alg == ecdh_anon; - Alg == psk; Alg == dhe_psk; Alg == rsa_psk; + Alg == psk; Alg == dhe_psk; Alg == ecdhe_psk; Alg == rsa_psk; Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon -> Params = ssl_handshake:decode_server_key(Keys, Alg, ssl:tls_version(Version)), @@ -542,6 +542,15 @@ certify(internal, #server_key_exchange{exchange_keys = Keys}, end end; +certify(internal, #certificate_request{}, + #state{role = client, negotiated_version = Version, + key_algorithm = Alg} = State, _) + when Alg == dh_anon; Alg == ecdh_anon; + Alg == psk; Alg == dhe_psk; Alg == ecdhe_psk; Alg == rsa_psk; + Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon -> + handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE), + Version, certify, State); + certify(internal, #certificate_request{} = CertRequest, #state{session = #session{own_certificate = Cert}, role = client, @@ -673,10 +682,11 @@ cipher(internal, #certificate_verify{signature = Signature, tls_handshake_history = Handshake } = State0, Connection) -> + TLSVersion = ssl:tls_version(Version), %% Use negotiated value if TLS-1.2 otherwhise return default - HashSign = negotiated_hashsign(CertHashSign, KexAlg, PublicKeyInfo, Version), + HashSign = negotiated_hashsign(CertHashSign, KexAlg, PublicKeyInfo, TLSVersion), case ssl_handshake:certificate_verify(Signature, PublicKeyInfo, - ssl:tls_version(Version), HashSign, MasterSecret, Handshake) of + TLSVersion, HashSign, MasterSecret, Handshake) of valid -> {Record, State} = Connection:next_record(State0), Connection:next_event(cipher, Record, @@ -1143,7 +1153,8 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName, port = Port, session = Session, user_application = {_Mon, Pid}, role = Role, socket_options = Opts, tracker = Tracker}) -> invalidate_session(Role, Host, Port, Session), - log_alert(SslOpts#ssl_options.log_alert, StateName, Alert), + log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(), + StateName, Alert#alert{role = opposite_role(Role)}), alert_user(Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, Role, Connection), {stop, normal}; @@ -1153,15 +1164,18 @@ handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert, {stop, {shutdown, peer_close}}; handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, StateName, - #state{ssl_options = SslOpts, renegotiation = {true, internal}} = State) -> - log_alert(SslOpts#ssl_options.log_alert, StateName, Alert), + #state{role = Role, ssl_options = SslOpts, protocol_cb = Connection, renegotiation = {true, internal}} = State) -> + log_alert(SslOpts#ssl_options.log_alert, Role, + Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), handle_normal_shutdown(Alert, StateName, State), {stop, {shutdown, peer_close}}; handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, StateName, - #state{ssl_options = SslOpts, renegotiation = {true, From}, + #state{role = Role, + ssl_options = SslOpts, renegotiation = {true, From}, protocol_cb = Connection} = State0) -> - log_alert(SslOpts#ssl_options.log_alert, StateName, Alert), + log_alert(SslOpts#ssl_options.log_alert, Role, + Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), gen_statem:reply(From, {error, renegotiation_rejected}), {Record, State} = Connection:next_record(State0), %% Go back to connection! @@ -1169,8 +1183,9 @@ handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, %% Gracefully log and ignore all other warning alerts handle_alert(#alert{level = ?WARNING} = Alert, StateName, - #state{ssl_options = SslOpts, protocol_cb = Connection} = State0) -> - log_alert(SslOpts#ssl_options.log_alert, StateName, Alert), + #state{ssl_options = SslOpts, protocol_cb = Connection, role = Role} = State0) -> + log_alert(SslOpts#ssl_options.log_alert, Role, + Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}), {Record, State} = Connection:next_record(State0), Connection:next_event(StateName, Record, State). @@ -1393,6 +1408,16 @@ certify_client_key_exchange(#client_dhe_psk_identity{} = ClientKey, PremasterSecret = ssl_handshake:premaster_secret(ClientKey, ServerDhPrivateKey, Params, PSKLookup), calculate_master_secret(PremasterSecret, State0, Connection, certify, cipher); + +certify_client_key_exchange(#client_ecdhe_psk_identity{} = ClientKey, + #state{diffie_hellman_keys = ServerEcDhPrivateKey, + ssl_options = + #ssl_options{user_lookup_fun = PSKLookup}} = State, + Connection) -> + PremasterSecret = + ssl_handshake:premaster_secret(ClientKey, ServerEcDhPrivateKey, PSKLookup), + calculate_master_secret(PremasterSecret, State, Connection, certify, cipher); + certify_client_key_exchange(#client_rsa_psk_identity{} = ClientKey, #state{private_key = Key, ssl_options = @@ -1412,6 +1437,7 @@ certify_server(#state{key_algorithm = Algo} = State, _) when Algo == dh_anon; Algo == ecdh_anon; Algo == psk; Algo == dhe_psk; + Algo == ecdhe_psk; Algo == srp_anon -> State; @@ -1518,6 +1544,28 @@ key_exchange(#state{role = server, key_algorithm = dhe_psk, State = Connection:queue_handshake(Msg, State0), State#state{diffie_hellman_keys = DHKeys}; +key_exchange(#state{role = server, key_algorithm = ecdhe_psk, + ssl_options = #ssl_options{psk_identity = PskIdentityHint}, + hashsign_algorithm = HashSignAlgo, + private_key = PrivateKey, + session = #session{ecc = ECCCurve}, + connection_states = ConnectionStates0, + negotiated_version = Version + } = State0, Connection) -> + ECDHKeys = public_key:generate_key(ECCCurve), + #{security_parameters := SecParams} = + ssl_record:pending_connection_state(ConnectionStates0, read), + #security_parameters{client_random = ClientRandom, + server_random = ServerRandom} = SecParams, + Msg = ssl_handshake:key_exchange(server, ssl:tls_version(Version), + {ecdhe_psk, + PskIdentityHint, ECDHKeys, + HashSignAlgo, ClientRandom, + ServerRandom, + PrivateKey}), + State = Connection:queue_handshake(Msg, State0), + State#state{diffie_hellman_keys = ECDHKeys}; + key_exchange(#state{role = server, key_algorithm = rsa_psk, ssl_options = #ssl_options{psk_identity = undefined}} = State, _) -> State; @@ -1616,6 +1664,17 @@ key_exchange(#state{role = client, {dhe_psk, SslOpts#ssl_options.psk_identity, DhPubKey}), Connection:queue_handshake(Msg, State0); + +key_exchange(#state{role = client, + ssl_options = SslOpts, + key_algorithm = ecdhe_psk, + negotiated_version = Version, + diffie_hellman_keys = ECDHKeys} = State0, Connection) -> + Msg = ssl_handshake:key_exchange(client, ssl:tls_version(Version), + {ecdhe_psk, + SslOpts#ssl_options.psk_identity, ECDHKeys}), + Connection:queue_handshake(Msg, State0); + key_exchange(#state{role = client, ssl_options = SslOpts, key_algorithm = rsa_psk, @@ -1671,6 +1730,12 @@ rsa_psk_key_exchange(Version, PskIdentity, PremasterSecret, rsa_psk_key_exchange(_, _, _, _) -> throw (?ALERT_REC(?FATAL,?HANDSHAKE_FAILURE, pub_key_is_not_rsa)). +request_client_cert(#state{key_algorithm = Alg} = State, _) + when Alg == dh_anon; Alg == ecdh_anon; + Alg == psk; Alg == dhe_psk; Alg == ecdhe_psk; Alg == rsa_psk; + Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon -> + State; + request_client_cert(#state{ssl_options = #ssl_options{verify = verify_peer, signature_algs = SupportedHashSigns}, connection_states = ConnectionStates0, @@ -1792,6 +1857,18 @@ calculate_secret(#server_dhe_psk_params{ calculate_master_secret(PremasterSecret, State#state{diffie_hellman_keys = Keys}, Connection, certify, certify); +calculate_secret(#server_ecdhe_psk_params{ + dh_params = #server_ecdh_params{curve = ECCurve}} = ServerKey, + #state{ssl_options = #ssl_options{user_lookup_fun = PSKLookup}} = + State=#state{session=Session}, Connection) -> + ECDHKeys = public_key:generate_key(ECCurve), + + PremasterSecret = ssl_handshake:premaster_secret(ServerKey, ECDHKeys, PSKLookup), + calculate_master_secret(PremasterSecret, + State#state{diffie_hellman_keys = ECDHKeys, + session = Session#session{ecc = ECCurve}}, + Connection, certify, certify); + calculate_secret(#server_srp_params{srp_n = Prime, srp_g = Generator} = ServerKey, #state{ssl_options = #ssl_options{srp_identity = SRPId}} = State, Connection) -> @@ -1876,6 +1953,7 @@ is_anonymous(Algo) when Algo == dh_anon; Algo == ecdh_anon; Algo == psk; Algo == dhe_psk; + Algo == ecdhe_psk; Algo == rsa_psk; Algo == srp_anon -> true; @@ -2370,18 +2448,22 @@ alert_user(Transport, Tracker, Socket, Active, Pid, From, Alert, Role, Connectio Transport, Socket, Connection, Tracker), ReasonCode}) end. -log_alert(true, Info, Alert) -> +log_alert(true, Role, ProtocolName, StateName, #alert{role = Role} = Alert) -> + Txt = ssl_alert:own_alert_txt(Alert), + error_logger:info_report(io_lib:format("~s ~p: In state ~p ~s\n", [ProtocolName, Role, StateName, Txt])); +log_alert(true, Role, ProtocolName, StateName, Alert) -> Txt = ssl_alert:alert_txt(Alert), - error_logger:format("SSL: ~p: ~s\n", [Info, Txt]); -log_alert(false, _, _) -> + error_logger:info_report(io_lib:format("~s ~p: In state ~p ~s\n", [ProtocolName, Role, StateName, Txt])); +log_alert(false, _, _, _, _) -> ok. handle_own_alert(Alert, Version, StateName, - #state{transport_cb = Transport, - socket = Socket, - protocol_cb = Connection, - connection_states = ConnectionStates, - ssl_options = SslOpts} = State) -> + #state{role = Role, + transport_cb = Transport, + socket = Socket, + protocol_cb = Connection, + connection_states = ConnectionStates, + ssl_options = SslOpts} = State) -> try %% Try to tell the other side {BinMsg, _} = Connection:encode_alert(Alert, Version, ConnectionStates), @@ -2390,7 +2472,7 @@ handle_own_alert(Alert, Version, StateName, ignore end, try %% Try to tell the local user - log_alert(SslOpts#ssl_options.log_alert, StateName, Alert), + log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(), StateName, Alert#alert{role = Role}), handle_normal_shutdown(Alert,StateName, State) catch _:_ -> ok diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 3cf466e78f..fc4181a760 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -227,6 +227,7 @@ certificate_request(CipherSuite, CertDbHandle, CertDbRef, HashSigns, Version) -> {ecdh, #'ECPrivateKey'{}} | {psk, binary()} | {dhe_psk, binary(), binary()} | + {ecdhe_psk, binary(), #'ECPrivateKey'{}} | {srp, {binary(), binary()}, #srp_user{}, {HashAlgo::atom(), SignAlgo::atom()}, binary(), binary(), public_key:private_key()}) -> #client_key_exchange{} | #server_key_exchange{}. @@ -264,6 +265,13 @@ key_exchange(client, _Version, {dhe_psk, Identity, PublicKey}) -> dh_public = PublicKey} }; +key_exchange(client, _Version, {ecdhe_psk, Identity, #'ECPrivateKey'{publicKey = ECPublicKey}}) -> + #client_key_exchange{ + exchange_keys = #client_ecdhe_psk_identity{ + identity = Identity, + dh_public = ECPublicKey} + }; + key_exchange(client, _Version, {psk_premaster_secret, PskIdentity, Secret, {_, PublicKey, _}}) -> EncPremasterSecret = encrypted_premaster_secret(Secret, PublicKey), @@ -310,6 +318,16 @@ key_exchange(server, Version, {dhe_psk, PskIdentityHint, {PublicKey, _}, enc_server_key_exchange(Version, ServerEDHPSKParams, HashSign, ClientRandom, ServerRandom, PrivateKey); +key_exchange(server, Version, {ecdhe_psk, PskIdentityHint, + #'ECPrivateKey'{publicKey = ECPublicKey, + parameters = ECCurve}, + HashSign, ClientRandom, ServerRandom, PrivateKey}) -> + ServerECDHEPSKParams = #server_ecdhe_psk_params{ + hint = PskIdentityHint, + dh_params = #server_ecdh_params{curve = ECCurve, public = ECPublicKey}}, + enc_server_key_exchange(Version, ServerECDHEPSKParams, HashSign, + ClientRandom, ServerRandom, PrivateKey); + key_exchange(server, Version, {srp, {PublicKey, _}, #srp_user{generator = Generator, prime = Prime, salt = Salt}, @@ -415,9 +433,11 @@ certify(#certificate{asn1_certificates = ASN1Certs}, CertDbHandle, CertDbRef, path_validation_alert(Reason) end catch - error:_ -> + error:{badmatch,{asn1, Asn1Reason}} -> %% ASN-1 decode of certificate somehow failed - ?ALERT_REC(?FATAL, ?CERTIFICATE_UNKNOWN, failed_to_decode_certificate) + ?ALERT_REC(?FATAL, ?CERTIFICATE_UNKNOWN, {failed_to_decode_certificate, Asn1Reason}); + error:OtherReason -> + ?ALERT_REC(?FATAL, ?INTERNAL_ERROR, {unexpected_error, OtherReason}) end. %%-------------------------------------------------------------------- @@ -530,14 +550,31 @@ premaster_secret(#server_dhe_psk_params{ LookupFun) -> PremasterSecret = premaster_secret(PublicDhKey, PrivateDhKey, Params), psk_secret(IdentityHint, LookupFun, PremasterSecret); + +premaster_secret(#server_ecdhe_psk_params{ + hint = IdentityHint, + dh_params = #server_ecdh_params{ + public = ECServerPubKey}}, + PrivateEcDhKey, + LookupFun) -> + PremasterSecret = premaster_secret(#'ECPoint'{point = ECServerPubKey}, PrivateEcDhKey), + psk_secret(IdentityHint, LookupFun, PremasterSecret); + premaster_secret({rsa_psk, PSKIdentity}, PSKLookup, RSAPremasterSecret) -> - psk_secret(PSKIdentity, PSKLookup, RSAPremasterSecret). + psk_secret(PSKIdentity, PSKLookup, RSAPremasterSecret); + +premaster_secret(#client_ecdhe_psk_identity{ + identity = PSKIdentity, + dh_public = PublicEcDhPoint}, PrivateEcDhKey, PSKLookup) -> + PremasterSecret = premaster_secret(#'ECPoint'{point = PublicEcDhPoint}, PrivateEcDhKey), + psk_secret(PSKIdentity, PSKLookup, PremasterSecret). premaster_secret(#client_dhe_psk_identity{ identity = PSKIdentity, dh_public = PublicDhKey}, PrivateKey, #'DHParameter'{} = Params, PSKLookup) -> PremasterSecret = premaster_secret(PublicDhKey, PrivateKey, Params), psk_secret(PSKIdentity, PSKLookup, PremasterSecret). + premaster_secret(#client_psk_identity{identity = PSKIdentity}, PSKLookup) -> psk_secret(PSKIdentity, PSKLookup); premaster_secret({psk, PSKIdentity}, PSKLookup) -> @@ -885,6 +922,7 @@ enc_server_key_exchange(Version, Params, {HashAlgo, SignAlgo}, | #client_ec_diffie_hellman_public{} | #client_psk_identity{} | #client_dhe_psk_identity{} + | #client_ecdhe_psk_identity{} | #client_rsa_psk_identity{} | #client_srp_public{}. %% @@ -1046,6 +1084,7 @@ dec_server_key(<<?UINT16(Len), PskIdentityHint:Len/binary, _/binary>> = KeyStruc params_bin = BinMsg, hashsign = HashSign, signature = Signature}; + dec_server_key(<<?UINT16(Len), IdentityHint:Len/binary, ?UINT16(PLen), P:PLen/binary, ?UINT16(GLen), G:GLen/binary, @@ -1060,6 +1099,22 @@ dec_server_key(<<?UINT16(Len), IdentityHint:Len/binary, params_bin = BinMsg, hashsign = HashSign, signature = Signature}; +dec_server_key(<<?UINT16(Len), IdentityHint:Len/binary, + ?BYTE(?NAMED_CURVE), ?UINT16(CurveID), + ?BYTE(PointLen), ECPoint:PointLen/binary, + _/binary>> = KeyStruct, + ?KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK, Version) -> + DHParams = #server_ecdh_params{ + curve = {namedCurve, tls_v1:enum_to_oid(CurveID)}, + public = ECPoint}, + Params = #server_ecdhe_psk_params{ + hint = IdentityHint, + dh_params = DHParams}, + {BinMsg, HashSign, Signature} = dec_server_key_params(Len + 2 + PointLen + 4, KeyStruct, Version), + #server_key_params{params = Params, + params_bin = BinMsg, + hashsign = HashSign, + signature = Signature}; dec_server_key(<<?UINT16(NLen), N:NLen/binary, ?UINT16(GLen), G:GLen/binary, ?BYTE(SLen), S:SLen/binary, @@ -1130,7 +1185,8 @@ filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, Acc KeyExchange == ecdh_anon; KeyExchange == srp_anon; KeyExchange == psk; - KeyExchange == dhe_psk -> + KeyExchange == dhe_psk; + KeyExchange == ecdhe_psk -> %% In this case hashsigns is not used as the kexchange is anonaymous filter_hashsigns(Suites, Algos, HashSigns, [Suite| Acc]). @@ -1494,6 +1550,8 @@ advertises_ec_ciphers([{ecdhe_rsa, _,_,_} | _]) -> true; advertises_ec_ciphers([{ecdh_anon, _,_,_} | _]) -> true; +advertises_ec_ciphers([{ecdhe_psk, _,_,_} | _]) -> + true; advertises_ec_ciphers([_| Rest]) -> advertises_ec_ciphers(Rest). @@ -1611,8 +1669,11 @@ path_validation_alert({bad_cert, unknown_critical_extension}) -> ?ALERT_REC(?FATAL, ?UNSUPPORTED_CERTIFICATE); path_validation_alert({bad_cert, {revoked, _}}) -> ?ALERT_REC(?FATAL, ?CERTIFICATE_REVOKED); -path_validation_alert({bad_cert, revocation_status_undetermined}) -> - ?ALERT_REC(?FATAL, ?BAD_CERTIFICATE); +%%path_validation_alert({bad_cert, revocation_status_undetermined}) -> +%% ?ALERT_REC(?FATAL, ?BAD_CERTIFICATE); +path_validation_alert({bad_cert, {revocation_status_undetermined, Details}}) -> + Alert = ?ALERT_REC(?FATAL, ?BAD_CERTIFICATE), + Alert#alert{reason = Details}; path_validation_alert({bad_cert, selfsigned_peer}) -> ?ALERT_REC(?FATAL, ?BAD_CERTIFICATE); path_validation_alert({bad_cert, unknown_ca}) -> @@ -1785,6 +1846,18 @@ encode_server_key(#server_dhe_psk_params{ YLen = byte_size(Y), <<?UINT16(Len), PskIdentityHint/binary, ?UINT16(PLen), P/binary, ?UINT16(GLen), G/binary, ?UINT16(YLen), Y/binary>>; +encode_server_key(Params = #server_ecdhe_psk_params{hint = undefined}) -> + encode_server_key(Params#server_ecdhe_psk_params{hint = <<>>}); +encode_server_key(#server_ecdhe_psk_params{ + hint = PskIdentityHint, + dh_params = #server_ecdh_params{ + curve = {namedCurve, ECCurve}, public = ECPubKey}}) -> + %%TODO: support arbitrary keys + Len = byte_size(PskIdentityHint), + KLen = size(ECPubKey), + <<?UINT16(Len), PskIdentityHint/binary, + ?BYTE(?NAMED_CURVE), ?UINT16((tls_v1:oid_to_enum(ECCurve))), + ?BYTE(KLen), ECPubKey/binary>>; encode_server_key(#server_srp_params{srp_n = N, srp_g = G, srp_s = S, srp_b = B}) -> NLen = byte_size(N), GLen = byte_size(G), @@ -1817,6 +1890,12 @@ encode_client_key(#client_dhe_psk_identity{identity = Id, dh_public = DHPublic}, Len = byte_size(Id), DHLen = byte_size(DHPublic), <<?UINT16(Len), Id/binary, ?UINT16(DHLen), DHPublic/binary>>; +encode_client_key(Identity = #client_ecdhe_psk_identity{identity = undefined}, Version) -> + encode_client_key(Identity#client_ecdhe_psk_identity{identity = <<"psk_identity">>}, Version); +encode_client_key(#client_ecdhe_psk_identity{identity = Id, dh_public = DHPublic}, _) -> + Len = byte_size(Id), + DHLen = byte_size(DHPublic), + <<?UINT16(Len), Id/binary, ?BYTE(DHLen), DHPublic/binary>>; encode_client_key(Identity = #client_rsa_psk_identity{identity = undefined}, Version) -> encode_client_key(Identity#client_rsa_psk_identity{identity = <<"psk_identity">>}, Version); encode_client_key(#client_rsa_psk_identity{identity = Id, exchange_keys = ExchangeKeys}, Version) -> @@ -1868,6 +1947,10 @@ dec_client_key(<<?UINT16(Len), Id:Len/binary, ?UINT16(DH_YLen), DH_Y:DH_YLen/binary>>, ?KEY_EXCHANGE_DHE_PSK, _) -> #client_dhe_psk_identity{identity = Id, dh_public = DH_Y}; +dec_client_key(<<?UINT16(Len), Id:Len/binary, + ?BYTE(DH_YLen), DH_Y:DH_YLen/binary>>, + ?KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK, _) -> + #client_ecdhe_psk_identity{identity = Id, dh_public = DH_Y}; dec_client_key(<<?UINT16(Len), Id:Len/binary, PKEPMS/binary>>, ?KEY_EXCHANGE_RSA_PSK, {3, 0}) -> #client_rsa_psk_identity{identity = Id, @@ -2045,6 +2128,8 @@ key_exchange_alg(psk) -> ?KEY_EXCHANGE_PSK; key_exchange_alg(dhe_psk) -> ?KEY_EXCHANGE_DHE_PSK; +key_exchange_alg(ecdhe_psk) -> + ?KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK; key_exchange_alg(rsa_psk) -> ?KEY_EXCHANGE_RSA_PSK; key_exchange_alg(Alg) @@ -2189,7 +2274,8 @@ crl_check(OtpCert, Check, CertDbHandle, CertDbRef, {Callback, CRLDbHandle}, _, C ssl_crl:trusted_cert_and_path(CRL, Issuer, {CertPath, DBInfo}) end, {CertDbHandle, CertDbRef}}}, - {update_crl, fun(DP, CRL) -> Callback:fresh_crl(DP, CRL) end} + {update_crl, fun(DP, CRL) -> Callback:fresh_crl(DP, CRL) end}, + {undetermined_details, true} ], case dps_and_crls(OtpCert, Callback, CRLDbHandle, ext) of no_dps -> @@ -2199,7 +2285,7 @@ crl_check(OtpCert, Check, CertDbHandle, CertDbRef, {Callback, CRLDbHandle}, _, C DpsAndCRLs -> %% This DP list may be empty if relevant CRLs existed %% but could not be retrived, will result in {bad_cert, revocation_status_undetermined} case public_key:pkix_crls_validate(OtpCert, DpsAndCRLs, Options) of - {bad_cert, revocation_status_undetermined} -> + {bad_cert, {revocation_status_undetermined, _}} -> crl_check_same_issuer(OtpCert, Check, dps_and_crls(OtpCert, Callback, CRLDbHandle, same_issuer), Options); Other -> @@ -2209,7 +2295,7 @@ crl_check(OtpCert, Check, CertDbHandle, CertDbRef, {Callback, CRLDbHandle}, _, C crl_check_same_issuer(OtpCert, best_effort, Dps, Options) -> case public_key:pkix_crls_validate(OtpCert, Dps, Options) of - {bad_cert, revocation_status_undetermined} -> + {bad_cert, {revocation_status_undetermined, _}} -> valid; Other -> Other @@ -2302,6 +2388,7 @@ is_acceptable_hash_sign({_, ecdsa} = Algos, ecdsa, ecdsa, ecdhe_ecdsa, Supported is_acceptable_hash_sign(_, _, _, KeyExAlgo, _) when KeyExAlgo == psk; KeyExAlgo == dhe_psk; + KeyExAlgo == ecdhe_psk; KeyExAlgo == srp_anon; KeyExAlgo == dh_anon; KeyExAlgo == ecdhe_anon diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index 324b7dbde3..a191fcf766 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -133,6 +133,7 @@ -define(KEY_EXCHANGE_DIFFIE_HELLMAN, 1). -define(KEY_EXCHANGE_EC_DIFFIE_HELLMAN, 6). -define(KEY_EXCHANGE_PSK, 2). +-define(KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK, 7). -define(KEY_EXCHANGE_DHE_PSK, 3). -define(KEY_EXCHANGE_RSA_PSK, 4). -define(KEY_EXCHANGE_SRP, 5). @@ -162,6 +163,11 @@ dh_params }). +-record(server_ecdhe_psk_params, { + hint, + dh_params + }). + -record(server_srp_params, { srp_n, %% opaque srp_N<1..2^16-1> srp_g, %% opaque srp_g<1..2^16-1> @@ -254,6 +260,11 @@ dh_public }). +-record(client_ecdhe_psk_identity, { + identity, + dh_public + }). + -record(client_rsa_psk_identity, { identity, exchange_keys diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl index ca9aaf4660..ef7c3de7af 100644 --- a/lib/ssl/src/ssl_manager.erl +++ b/lib/ssl/src/ssl_manager.erl @@ -563,7 +563,7 @@ server_register_session(Port, Session, #state{session_cache_server_max = Max, do_register_session(Key, Session, Max, Pid, Cache, CacheCb) -> try CacheCb:size(Cache) of - Max -> + Size when Size >= Max -> invalidate_session_cache(Pid, CacheCb, Cache); _ -> CacheCb:update(Cache, Key, Session), diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 352874c77d..e3ffbea3d3 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -56,7 +56,7 @@ reinit_handshake_data/1, select_sni_extension/1]). %% Alert and close handling --export([send_alert/2, close/5]). +-export([send_alert/2, close/5, protocol_name/0]). %% Data handling -export([passive_receive/2, next_record_if_active/1, handle_common_event/4, send/3, @@ -164,6 +164,8 @@ encode_data(Data, Version, ConnectionStates0)-> encode_alert(#alert{} = Alert, Version, ConnectionStates) -> tls_record:encode_alert_record(Alert, Version, ConnectionStates). +protocol_name() -> + "TLS". %%==================================================================== %% tls_connection_sup API %%==================================================================== @@ -719,7 +721,7 @@ close(downgrade, _,_,_,_) -> %% Other close(_, Socket, Transport, _,_) -> Transport:close(Socket). - + convert_state(#state{ssl_options = Options} = State, up, "5.3.5", "5.3.6") -> State#state{ssl_options = convert_options_partial_chain(Options, up)}; convert_state(#state{ssl_options = Options} = State, down, "5.3.6", "5.3.5") -> diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 558be6d642..c7e2f402af 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -56,7 +56,6 @@ MODULES = \ ssl_upgrade_SUITE\ ssl_sni_SUITE \ make_certs\ - erl_make_certs\ x509_test diff --git a/lib/ssl/test/erl_make_certs.erl b/lib/ssl/test/erl_make_certs.erl deleted file mode 100644 index 3ab6222780..0000000000 --- a/lib/ssl/test/erl_make_certs.erl +++ /dev/null @@ -1,477 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2011-2017. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - -%% Create test certificates - --module(erl_make_certs). --include_lib("public_key/include/public_key.hrl"). - --export([make_cert/1, gen_rsa/1, verify_signature/3, write_pem/3]). --compile(export_all). - -%%-------------------------------------------------------------------- -%% @doc Create and return a der encoded certificate -%% Option Default -%% ------------------------------------------------------- -%% digest sha1 -%% validity {date(), date() + week()} -%% version 3 -%% subject [] list of the following content -%% {name, Name} -%% {email, Email} -%% {city, City} -%% {state, State} -%% {org, Org} -%% {org_unit, OrgUnit} -%% {country, Country} -%% {serial, Serial} -%% {title, Title} -%% {dnQualifer, DnQ} -%% issuer = {Issuer, IssuerKey} true (i.e. a ca cert is created) -%% (obs IssuerKey migth be {Key, Password} -%% key = KeyFile|KeyBin|rsa|dsa|ec Subject PublicKey rsa, dsa or ec generates key -%% -%% -%% (OBS: The generated keys are for testing only) -%% @spec ([{::atom(), ::term()}]) -> {Cert::binary(), Key::binary()} -%% @end -%%-------------------------------------------------------------------- - -make_cert(Opts) -> - SubjectPrivateKey = get_key(Opts), - {TBSCert, IssuerKey} = make_tbs(SubjectPrivateKey, Opts), - Cert = public_key:pkix_sign(TBSCert, IssuerKey), - true = verify_signature(Cert, IssuerKey, undef), %% verify that the keys where ok - {Cert, encode_key(SubjectPrivateKey)}. - -%%-------------------------------------------------------------------- -%% @doc Writes pem files in Dir with FileName ++ ".pem" and FileName ++ "_key.pem" -%% @spec (::string(), ::string(), {Cert,Key}) -> ok -%% @end -%%-------------------------------------------------------------------- -write_pem(Dir, FileName, {Cert, Key = {_,_,not_encrypted}}) when is_binary(Cert) -> - ok = der_to_pem(filename:join(Dir, FileName ++ ".pem"), - [{'Certificate', Cert, not_encrypted}]), - ok = der_to_pem(filename:join(Dir, FileName ++ "_key.pem"), [Key]). - -%%-------------------------------------------------------------------- -%% @doc Creates a rsa key (OBS: for testing only) -%% the size are in bytes -%% @spec (::integer()) -> {::atom(), ::binary(), ::opaque()} -%% @end -%%-------------------------------------------------------------------- -gen_rsa(Size) when is_integer(Size) -> - Key = gen_rsa2(Size), - {Key, encode_key(Key)}. - -%%-------------------------------------------------------------------- -%% @doc Creates a dsa key (OBS: for testing only) -%% the sizes are in bytes -%% @spec (::integer()) -> {::atom(), ::binary(), ::opaque()} -%% @end -%%-------------------------------------------------------------------- -gen_dsa(LSize,NSize) when is_integer(LSize), is_integer(NSize) -> - Key = gen_dsa2(LSize, NSize), - {Key, encode_key(Key)}. - -%%-------------------------------------------------------------------- -%% @doc Creates a ec key (OBS: for testing only) -%% the sizes are in bytes -%% @spec (::integer()) -> {::atom(), ::binary(), ::opaque()} -%% @end -%%-------------------------------------------------------------------- -gen_ec(Curve) when is_atom(Curve) -> - Key = gen_ec2(Curve), - {Key, encode_key(Key)}. - -%%-------------------------------------------------------------------- -%% @doc Verifies cert signatures -%% @spec (::binary(), ::tuple()) -> ::boolean() -%% @end -%%-------------------------------------------------------------------- -verify_signature(DerEncodedCert, DerKey, _KeyParams) -> - Key = decode_key(DerKey), - case Key of - #'RSAPrivateKey'{modulus=Mod, publicExponent=Exp} -> - public_key:pkix_verify(DerEncodedCert, - #'RSAPublicKey'{modulus=Mod, publicExponent=Exp}); - #'DSAPrivateKey'{p=P, q=Q, g=G, y=Y} -> - public_key:pkix_verify(DerEncodedCert, {Y, #'Dss-Parms'{p=P, q=Q, g=G}}); - #'ECPrivateKey'{version = _Version, privateKey = _PrivKey, - parameters = Params, publicKey = PubKey} -> - public_key:pkix_verify(DerEncodedCert, {#'ECPoint'{point = PubKey}, Params}) - end. - -%%%%%%%%%%%%%%%%%%%%%%%%% Implementation %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -get_key(Opts) -> - case proplists:get_value(key, Opts) of - undefined -> make_key(rsa, Opts); - rsa -> make_key(rsa, Opts); - dsa -> make_key(dsa, Opts); - ec -> make_key(ec, Opts); - Key -> - Password = proplists:get_value(password, Opts, no_passwd), - decode_key(Key, Password) - end. - -decode_key({Key, Pw}) -> - decode_key(Key, Pw); -decode_key(Key) -> - decode_key(Key, no_passwd). - - -decode_key(#'RSAPublicKey'{} = Key,_) -> - Key; -decode_key(#'RSAPrivateKey'{} = Key,_) -> - Key; -decode_key(#'DSAPrivateKey'{} = Key,_) -> - Key; -decode_key(#'ECPrivateKey'{} = Key,_) -> - Key; -decode_key(PemEntry = {_,_,_}, Pw) -> - public_key:pem_entry_decode(PemEntry, Pw); -decode_key(PemBin, Pw) -> - [KeyInfo] = public_key:pem_decode(PemBin), - decode_key(KeyInfo, Pw). - -encode_key(Key = #'RSAPrivateKey'{}) -> - {ok, Der} = 'OTP-PUB-KEY':encode('RSAPrivateKey', Key), - {'RSAPrivateKey', Der, not_encrypted}; -encode_key(Key = #'DSAPrivateKey'{}) -> - {ok, Der} = 'OTP-PUB-KEY':encode('DSAPrivateKey', Key), - {'DSAPrivateKey', Der, not_encrypted}; -encode_key(Key = #'ECPrivateKey'{}) -> - {ok, Der} = 'OTP-PUB-KEY':encode('ECPrivateKey', Key), - {'ECPrivateKey', Der, not_encrypted}. - -make_tbs(SubjectKey, Opts) -> - Version = list_to_atom("v"++integer_to_list(proplists:get_value(version, Opts, 3))), - - IssuerProp = proplists:get_value(issuer, Opts, true), - {Issuer, IssuerKey} = issuer(IssuerProp, Opts, SubjectKey), - - {Algo, Parameters} = sign_algorithm(IssuerKey, Opts), - - SignAlgo = #'SignatureAlgorithm'{algorithm = Algo, - parameters = Parameters}, - Subject = case IssuerProp of - true -> %% Is a Root Ca - Issuer; - _ -> - subject(proplists:get_value(subject, Opts),false) - end, - - {#'OTPTBSCertificate'{serialNumber = trunc(rand:uniform()*100000000)*10000 + 1, - signature = SignAlgo, - issuer = Issuer, - validity = validity(Opts), - subject = Subject, - subjectPublicKeyInfo = publickey(SubjectKey), - version = Version, - extensions = extensions(Opts) - }, IssuerKey}. - -issuer(true, Opts, SubjectKey) -> - %% Self signed - {subject(proplists:get_value(subject, Opts), true), SubjectKey}; -issuer({Issuer, IssuerKey}, _Opts, _SubjectKey) when is_binary(Issuer) -> - {issuer_der(Issuer), decode_key(IssuerKey)}; -issuer({File, IssuerKey}, _Opts, _SubjectKey) when is_list(File) -> - {ok, [{cert, Cert, _}|_]} = pem_to_der(File), - {issuer_der(Cert), decode_key(IssuerKey)}. - -issuer_der(Issuer) -> - Decoded = public_key:pkix_decode_cert(Issuer, otp), - #'OTPCertificate'{tbsCertificate=Tbs} = Decoded, - #'OTPTBSCertificate'{subject=Subject} = Tbs, - Subject. - -subject(undefined, IsRootCA) -> - User = if IsRootCA -> "RootCA"; true -> os:getenv("USER", "test_user") end, - Opts = [{email, User ++ "@erlang.org"}, - {name, User}, - {city, "Stockholm"}, - {country, "SE"}, - {org, "erlang"}, - {org_unit, "testing dep"}], - subject(Opts); -subject(Opts, _) -> - subject(Opts). - -subject(SubjectOpts) when is_list(SubjectOpts) -> - Encode = fun(Opt) -> - {Type,Value} = subject_enc(Opt), - [#'AttributeTypeAndValue'{type=Type, value=Value}] - end, - {rdnSequence, [Encode(Opt) || Opt <- SubjectOpts]}. - -%% Fill in the blanks -subject_enc({name, Name}) -> {?'id-at-commonName', {printableString, Name}}; -subject_enc({email, Email}) -> {?'id-emailAddress', Email}; -subject_enc({city, City}) -> {?'id-at-localityName', {printableString, City}}; -subject_enc({state, State}) -> {?'id-at-stateOrProvinceName', {printableString, State}}; -subject_enc({org, Org}) -> {?'id-at-organizationName', {printableString, Org}}; -subject_enc({org_unit, OrgUnit}) -> {?'id-at-organizationalUnitName', {printableString, OrgUnit}}; -subject_enc({country, Country}) -> {?'id-at-countryName', Country}; -subject_enc({serial, Serial}) -> {?'id-at-serialNumber', Serial}; -subject_enc({title, Title}) -> {?'id-at-title', {printableString, Title}}; -subject_enc({dnQualifer, DnQ}) -> {?'id-at-dnQualifier', DnQ}; -subject_enc(Other) -> Other. - - -extensions(Opts) -> - case proplists:get_value(extensions, Opts, []) of - false -> - asn1_NOVALUE; - Exts -> - lists:flatten([extension(Ext) || Ext <- default_extensions(Exts)]) - end. - -default_extensions(Exts) -> - Def = [{key_usage,undefined}, - {subject_altname, undefined}, - {issuer_altname, undefined}, - {basic_constraints, default}, - {name_constraints, undefined}, - {policy_constraints, undefined}, - {ext_key_usage, undefined}, - {inhibit_any, undefined}, - {auth_key_id, undefined}, - {subject_key_id, undefined}, - {policy_mapping, undefined}], - Filter = fun({Key, _}, D) -> lists:keydelete(Key, 1, D) end, - Exts ++ lists:foldl(Filter, Def, Exts). - -extension({_, undefined}) -> []; -extension({basic_constraints, Data}) -> - case Data of - default -> - #'Extension'{extnID = ?'id-ce-basicConstraints', - extnValue = #'BasicConstraints'{cA=true}, - critical=true}; - false -> - []; - Len when is_integer(Len) -> - #'Extension'{extnID = ?'id-ce-basicConstraints', - extnValue = #'BasicConstraints'{cA=true, pathLenConstraint=Len}, - critical=true}; - _ -> - #'Extension'{extnID = ?'id-ce-basicConstraints', - extnValue = Data} - end; -extension({Id, Data, Critical}) -> - #'Extension'{extnID = Id, extnValue = Data, critical = Critical}. - - -publickey(#'RSAPrivateKey'{modulus=N, publicExponent=E}) -> - Public = #'RSAPublicKey'{modulus=N, publicExponent=E}, - Algo = #'PublicKeyAlgorithm'{algorithm= ?rsaEncryption, parameters='NULL'}, - #'OTPSubjectPublicKeyInfo'{algorithm = Algo, - subjectPublicKey = Public}; -publickey(#'DSAPrivateKey'{p=P, q=Q, g=G, y=Y}) -> - Algo = #'PublicKeyAlgorithm'{algorithm= ?'id-dsa', - parameters={params, #'Dss-Parms'{p=P, q=Q, g=G}}}, - #'OTPSubjectPublicKeyInfo'{algorithm = Algo, subjectPublicKey = Y}; -publickey(#'ECPrivateKey'{version = _Version, - privateKey = _PrivKey, - parameters = Params, - publicKey = PubKey}) -> - Algo = #'PublicKeyAlgorithm'{algorithm= ?'id-ecPublicKey', parameters=Params}, - #'OTPSubjectPublicKeyInfo'{algorithm = Algo, - subjectPublicKey = #'ECPoint'{point = PubKey}}. - -validity(Opts) -> - DefFrom0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())-1), - DefTo0 = calendar:gregorian_days_to_date(calendar:date_to_gregorian_days(date())+7), - {DefFrom, DefTo} = proplists:get_value(validity, Opts, {DefFrom0, DefTo0}), - Format = fun({Y,M,D}) -> lists:flatten(io_lib:format("~w~2..0w~2..0w000000Z",[Y,M,D])) end, - #'Validity'{notBefore={generalTime, Format(DefFrom)}, - notAfter ={generalTime, Format(DefTo)}}. - -sign_algorithm(#'RSAPrivateKey'{}, Opts) -> - Type = case proplists:get_value(digest, Opts, sha1) of - sha1 -> ?'sha1WithRSAEncryption'; - sha512 -> ?'sha512WithRSAEncryption'; - sha384 -> ?'sha384WithRSAEncryption'; - sha256 -> ?'sha256WithRSAEncryption'; - md5 -> ?'md5WithRSAEncryption'; - md2 -> ?'md2WithRSAEncryption' - end, - {Type, 'NULL'}; -sign_algorithm(#'DSAPrivateKey'{p=P, q=Q, g=G}, _Opts) -> - {?'id-dsa-with-sha1', {params,#'Dss-Parms'{p=P, q=Q, g=G}}}; -sign_algorithm(#'ECPrivateKey'{parameters = Parms}, Opts) -> - Type = case proplists:get_value(digest, Opts, sha1) of - sha1 -> ?'ecdsa-with-SHA1'; - sha512 -> ?'ecdsa-with-SHA512'; - sha384 -> ?'ecdsa-with-SHA384'; - sha256 -> ?'ecdsa-with-SHA256' - end, - {Type, Parms}. - -make_key(rsa, _Opts) -> - %% (OBS: for testing only) - gen_rsa2(64); -make_key(dsa, _Opts) -> - gen_dsa2(128, 20); %% Bytes i.e. {1024, 160} -make_key(ec, _Opts) -> - %% (OBS: for testing only) - CurveOid = hd(tls_v1:ecc_curves(0)), - NamedCurve = pubkey_cert_records:namedCurves(CurveOid), - gen_ec2(NamedCurve). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% RSA key generation (OBS: for testing only) -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - --define(SMALL_PRIMES, [65537,97,89,83,79,73,71,67,61,59,53, - 47,43,41,37,31,29,23,19,17,13,11,7,5,3]). - -gen_rsa2(Size) -> - P = prime(Size), - Q = prime(Size), - N = P*Q, - Tot = (P - 1) * (Q - 1), - [E|_] = lists:dropwhile(fun(Candidate) -> (Tot rem Candidate) == 0 end, ?SMALL_PRIMES), - {D1,D2} = extended_gcd(E, Tot), - D = erlang:max(D1,D2), - case D < E of - true -> - gen_rsa2(Size); - false -> - {Co1,Co2} = extended_gcd(Q, P), - Co = erlang:max(Co1,Co2), - #'RSAPrivateKey'{version = 'two-prime', - modulus = N, - publicExponent = E, - privateExponent = D, - prime1 = P, - prime2 = Q, - exponent1 = D rem (P-1), - exponent2 = D rem (Q-1), - coefficient = Co - } - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% DSA key generation (OBS: for testing only) -%% See http://en.wikipedia.org/wiki/Digital_Signature_Algorithm -%% and the fips_186-3.pdf -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -gen_dsa2(LSize, NSize) -> - Q = prime(NSize), %% Choose N-bit prime Q - X0 = prime(LSize), - P0 = prime((LSize div 2) +1), - - %% Choose L-bit prime modulus P such that p-1 is a multiple of q. - case dsa_search(X0 div (2*Q*P0), P0, Q, 1000) of - error -> - gen_dsa2(LSize, NSize); - P -> - G = crypto:mod_pow(2, (P-1) div Q, P), % Choose G a number whose multiplicative order modulo p is q. - %% such that This may be done by setting g = h^(p-1)/q mod p, commonly h=2 is used. - - X = prime(20), %% Choose x by some random method, where 0 < x < q. - Y = crypto:mod_pow(G, X, P), %% Calculate y = g^x mod p. - - #'DSAPrivateKey'{version=0, p = P, q = Q, - g = crypto:bytes_to_integer(G), y = crypto:bytes_to_integer(Y), x = X} - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% EC key generation (OBS: for testing only) -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -gen_ec2(CurveId) -> - {PubKey, PrivKey} = crypto:generate_key(ecdh, CurveId), - - #'ECPrivateKey'{version = 1, - privateKey = PrivKey, - parameters = {namedCurve, pubkey_cert_records:namedCurves(CurveId)}, - publicKey = PubKey}. - -%% See fips_186-3.pdf -dsa_search(T, P0, Q, Iter) when Iter > 0 -> - P = 2*T*Q*P0 + 1, - case is_prime(P, 50) of - true -> P; - false -> dsa_search(T+1, P0, Q, Iter-1) - end; -dsa_search(_,_,_,_) -> - error. - - -%%%%%%% Crypto Math %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -prime(ByteSize) -> - Rand = odd_rand(ByteSize), - prime_odd(Rand, 0). - -prime_odd(Rand, N) -> - case is_prime(Rand, 50) of - true -> - Rand; - false -> - prime_odd(Rand+2, N+1) - end. - -%% see http://en.wikipedia.org/wiki/Fermat_primality_test -is_prime(_, 0) -> true; -is_prime(Candidate, Test) -> - CoPrime = odd_rand(10000, Candidate), - Result = crypto:mod_pow(CoPrime, Candidate, Candidate) , - is_prime(CoPrime, crypto:bytes_to_integer(Result), Candidate, Test). - -is_prime(CoPrime, CoPrime, Candidate, Test) -> - is_prime(Candidate, Test-1); -is_prime(_,_,_,_) -> - false. - -odd_rand(Size) -> - Min = 1 bsl (Size*8-1), - Max = (1 bsl (Size*8))-1, - odd_rand(Min, Max). - -odd_rand(Min,Max) -> - Rand = crypto:rand_uniform(Min,Max), - case Rand rem 2 of - 0 -> - Rand + 1; - _ -> - Rand - end. - -extended_gcd(A, B) -> - case A rem B of - 0 -> - {0, 1}; - N -> - {X, Y} = extended_gcd(B, N), - {Y, X-Y*(A div B)} - end. - -pem_to_der(File) -> - {ok, PemBin} = file:read_file(File), - public_key:pem_decode(PemBin). - -der_to_pem(File, Entries) -> - PemBin = public_key:pem_encode(Entries), - file:write_file(File, PemBin). - diff --git a/lib/ssl/test/ssl_ECC_SUITE.erl b/lib/ssl/test/ssl_ECC_SUITE.erl index 0fbb0bb79a..64e8042b25 100644 --- a/lib/ssl/test/ssl_ECC_SUITE.erl +++ b/lib/ssl/test/ssl_ECC_SUITE.erl @@ -36,7 +36,9 @@ all() -> [ {group, 'tlsv1.2'}, {group, 'tlsv1.1'}, - {group, 'tlsv1'} + {group, 'tlsv1'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} ]. groups() -> @@ -44,6 +46,8 @@ groups() -> {'tlsv1.2', [], all_versions_groups()}, {'tlsv1.1', [], all_versions_groups()}, {'tlsv1', [], all_versions_groups()}, + {'dtlsv1.2', [], all_versions_groups()}, + {'dtlsv1', [], all_versions_groups()}, {'erlang_server', [], openssl_key_cert_combinations()}, %%{'erlang_client', [], openssl_key_cert_combinations()}, {'erlang', [], key_cert_combinations() ++ misc() @@ -196,8 +200,14 @@ common_init_per_group(GroupName, Config) -> openssl_check(GroupName, Config) end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config0) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + Config = ssl_test_lib:clean_tls_version(Config0), + proplists:delete(tls_version, Config); + false -> + Config0 + end. %%-------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl index 158b3524ac..055f05a900 100644 --- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl @@ -35,14 +35,19 @@ all() -> [{group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, - {group, 'sslv3'}]. + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. groups() -> [ {'tlsv1.2', [], alpn_tests()}, {'tlsv1.1', [], alpn_tests()}, {'tlsv1', [], alpn_tests()}, - {'sslv3', [], alpn_not_supported()} + {'sslv3', [], alpn_not_supported()}, + {'dtlsv1.2', [], alpn_tests() -- [client_renegotiate]}, + {'dtlsv1', [], alpn_tests() -- [client_renegotiate]} ]. alpn_tests() -> @@ -67,13 +72,12 @@ alpn_not_supported() -> alpn_not_supported_server ]. -init_per_suite(Config) -> +init_per_suite(Config0) -> catch crypto:stop(), try crypto:start() of ok -> ssl_test_lib:clean_start(), - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config), - proplists:get_value(priv_dir, Config)), + Config = ssl_test_lib:make_rsa_cert(Config0), ssl_test_lib:cert_options(Config) catch _:_ -> {skip, "Crypto did not start"} @@ -90,8 +94,7 @@ init_per_group(GroupName, Config) -> true -> case ssl_test_lib:sufficient_crypto_support(GroupName) of true -> - ssl_test_lib:init_tls_version(GroupName, Config), - Config; + ssl_test_lib:init_tls_version(GroupName, Config); false -> {skip, "Missing crypto support"} end; @@ -100,8 +103,14 @@ init_per_group(GroupName, Config) -> Config end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. + init_per_testcase(_TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), @@ -116,26 +125,29 @@ end_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- empty_protocols_are_not_allowed(Config) when is_list(Config) -> + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), {error, {options, {alpn_preferred_protocols, {invalid_protocol, <<>>}}}} = (catch ssl:listen(9443, - [{alpn_preferred_protocols, [<<"foo/1">>, <<"">>]}])), + [{alpn_preferred_protocols, [<<"foo/1">>, <<"">>]}| ServerOpts])), {error, {options, {alpn_advertised_protocols, {invalid_protocol, <<>>}}}} = (catch ssl:connect({127,0,0,1}, 9443, - [{alpn_advertised_protocols, [<<"foo/1">>, <<"">>]}])). + [{alpn_advertised_protocols, [<<"foo/1">>, <<"">>]} | ServerOpts])). %-------------------------------------------------------------------------------- protocols_must_be_a_binary_list(Config) when is_list(Config) -> + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), Option1 = {alpn_preferred_protocols, hello}, - {error, {options, Option1}} = (catch ssl:listen(9443, [Option1])), + {error, {options, Option1}} = (catch ssl:listen(9443, [Option1 | ServerOpts])), Option2 = {alpn_preferred_protocols, [<<"foo/1">>, hello]}, {error, {options, {alpn_preferred_protocols, {invalid_protocol, hello}}}} - = (catch ssl:listen(9443, [Option2])), + = (catch ssl:listen(9443, [Option2 | ServerOpts])), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), Option3 = {alpn_advertised_protocols, hello}, - {error, {options, Option3}} = (catch ssl:connect({127,0,0,1}, 9443, [Option3])), + {error, {options, Option3}} = (catch ssl:connect({127,0,0,1}, 9443, [Option3 | ClientOpts])), Option4 = {alpn_advertised_protocols, [<<"foo/1">>, hello]}, {error, {options, {alpn_advertised_protocols, {invalid_protocol, hello}}}} - = (catch ssl:connect({127,0,0,1}, 9443, [Option4])). + = (catch ssl:connect({127,0,0,1}, 9443, [Option4 | ClientOpts])). %-------------------------------------------------------------------------------- @@ -226,9 +238,9 @@ client_alpn_and_server_alpn_npn(Config) when is_list(Config) -> client_renegotiate(Config) when is_list(Config) -> Data = "hello world", - ClientOpts0 = proplists:get_value(client_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), ClientOpts = [{alpn_advertised_protocols, [<<"http/1.0">>]}] ++ ClientOpts0, - ServerOpts0 = proplists:get_value(server_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), ServerOpts = [{alpn_preferred_protocols, [<<"spdy/2">>, <<"http/1.1">>, <<"http/1.0">>]}] ++ ServerOpts0, ExpectedProtocol = {ok, <<"http/1.0">>}, @@ -250,9 +262,9 @@ client_renegotiate(Config) when is_list(Config) -> %-------------------------------------------------------------------------------- session_reused(Config) when is_list(Config)-> - ClientOpts0 = proplists:get_value(client_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), ClientOpts = [{alpn_advertised_protocols, [<<"http/1.0">>]}] ++ ClientOpts0, - ServerOpts0 = proplists:get_value(server_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), ServerOpts = [{alpn_preferred_protocols, [<<"spdy/2">>, <<"http/1.1">>, <<"http/1.0">>]}] ++ ServerOpts0, {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -299,7 +311,7 @@ session_reused(Config) when is_list(Config)-> %-------------------------------------------------------------------------------- alpn_not_supported_client(Config) when is_list(Config) -> - ClientOpts0 = proplists:get_value(client_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), PrefProtocols = {client_preferred_next_protocols, {client, [<<"http/1.0">>], <<"http/1.1">>}}, ClientOpts = [PrefProtocols] ++ ClientOpts0, @@ -315,7 +327,7 @@ alpn_not_supported_client(Config) when is_list(Config) -> %-------------------------------------------------------------------------------- alpn_not_supported_server(Config) when is_list(Config)-> - ServerOpts0 = proplists:get_value(server_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), AdvProtocols = {next_protocols_advertised, [<<"spdy/2">>, <<"http/1.1">>, <<"http/1.0">>]}, ServerOpts = [AdvProtocols] ++ ServerOpts0, @@ -326,8 +338,8 @@ alpn_not_supported_server(Config) when is_list(Config)-> %%-------------------------------------------------------------------- run_failing_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedResult) -> - ClientOpts = ClientExtraOpts ++ proplists:get_value(client_opts, Config), - ServerOpts = ServerExtraOpts ++ proplists:get_value(server_opts, Config), + ClientOpts = ClientExtraOpts ++ ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ServerExtraOpts ++ ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -346,9 +358,9 @@ run_failing_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedResult) run_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedProtocol) -> Data = "hello world", - ClientOpts0 = proplists:get_value(client_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), ClientOpts = ClientExtraOpts ++ ClientOpts0, - ServerOpts0 = proplists:get_value(server_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), ServerOpts = ServerExtraOpts ++ ServerOpts0, {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 407152aa75..2bbb03a854 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -53,8 +53,7 @@ all() -> {group, options_tls}, {group, session}, {group, 'dtlsv1.2'}, - %% {group, 'dtlsv1'}, Breaks dtls in cert_verify_SUITE enable later when - %% problem is identified and fixed + {group, 'dtlsv1'}, {group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, @@ -291,8 +290,13 @@ init_per_group(GroupName, Config) -> end end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. %%-------------------------------------------------------------------- init_per_testcase(Case, Config) when Case == unordered_protocol_versions_client; @@ -360,6 +364,8 @@ init_per_testcase(TestCase, Config) when TestCase == psk_cipher_suites; TestCase == psk_with_hint_cipher_suites; TestCase == ciphers_rsa_signed_certs; TestCase == ciphers_rsa_signed_certs_openssl_names; + TestCase == ciphers_ecdh_rsa_signed_certs_openssl_names; + TestCase == ciphers_ecdh_rsa_signed_certs; TestCase == ciphers_dsa_signed_certs; TestCase == ciphers_dsa_signed_certs_openssl_names; TestCase == anonymous_cipher_suites; @@ -368,6 +374,11 @@ init_per_testcase(TestCase, Config) when TestCase == psk_cipher_suites; TestCase == anonymous_cipher_suites; TestCase == psk_anon_cipher_suites; TestCase == psk_anon_with_hint_cipher_suites; + TestCase == srp_cipher_suites, + TestCase == srp_anon_cipher_suites, + TestCase == srp_dsa_cipher_suites, + TestCase == des_rsa_cipher_suites, + TestCase == des_ecdh_rsa_cipher_suites, TestCase == versions_option, TestCase == tls_tcp_connect_big -> ssl_test_lib:ct_log_supported_protocol_versions(Config), @@ -386,22 +397,27 @@ init_per_testcase(reuse_session, Config) -> init_per_testcase(rizzo, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), - ct:timetrap({seconds, 40}), + ct:timetrap({seconds, 60}), + Config; + +init_per_testcase(no_rizzo_rc4, Config) -> + ssl_test_lib:ct_log_supported_protocol_versions(Config), + ct:timetrap({seconds, 60}), Config; init_per_testcase(rizzo_one_n_minus_one, Config) -> ct:log("TLS/SSL version ~p~n ", [tls_record:supported_protocol_versions()]), - ct:timetrap({seconds, 40}), + ct:timetrap({seconds, 60}), rizzo_add_mitigation_option(one_n_minus_one, Config); init_per_testcase(rizzo_zero_n, Config) -> ct:log("TLS/SSL version ~p~n ", [tls_record:supported_protocol_versions()]), - ct:timetrap({seconds, 40}), + ct:timetrap({seconds, 60}), rizzo_add_mitigation_option(zero_n, Config); init_per_testcase(rizzo_disabled, Config) -> ct:log("TLS/SSL version ~p~n ", [tls_record:supported_protocol_versions()]), - ct:timetrap({seconds, 40}), + ct:timetrap({seconds, 60}), rizzo_add_mitigation_option(disabled, Config); init_per_testcase(prf, Config) -> @@ -511,7 +527,7 @@ alerts() -> [{doc, "Test ssl_alert:alert_txt/1"}]. alerts(Config) when is_list(Config) -> Descriptions = [?CLOSE_NOTIFY, ?UNEXPECTED_MESSAGE, ?BAD_RECORD_MAC, - ?DECRYPTION_FAILED, ?RECORD_OVERFLOW, ?DECOMPRESSION_FAILURE, + ?DECRYPTION_FAILED_RESERVED, ?RECORD_OVERFLOW, ?DECOMPRESSION_FAILURE, ?HANDSHAKE_FAILURE, ?BAD_CERTIFICATE, ?UNSUPPORTED_CERTIFICATE, ?CERTIFICATE_REVOKED,?CERTIFICATE_EXPIRED, ?CERTIFICATE_UNKNOWN, ?ILLEGAL_PARAMETER, ?UNKNOWN_CA, ?ACCESS_DENIED, ?DECODE_ERROR, @@ -2308,20 +2324,16 @@ tls_shutdown_error(Config) when is_list(Config) -> ciphers_rsa_signed_certs() -> [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. -ciphers_rsa_signed_certs(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), +ciphers_rsa_signed_certs(Config) when is_list(Config) -> Ciphers = ssl_test_lib:rsa_suites(crypto), - ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), - run_suites(Ciphers, Version, Config, rsa). + run_suites(Ciphers, Config, rsa). %%------------------------------------------------------------------- ciphers_rsa_signed_certs_openssl_names() -> [{doc,"Test all rsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:openssl_rsa_suites(crypto), - ct:log("tls1 openssl cipher suites ~p~n", [Ciphers]), - run_suites(Ciphers, Version, Config, rsa). + Ciphers = ssl_test_lib:openssl_rsa_suites(), + run_suites(Ciphers, Config, rsa). %%------------------------------------------------------------------- ciphers_dsa_signed_certs() -> @@ -2329,120 +2341,104 @@ ciphers_dsa_signed_certs() -> ciphers_dsa_signed_certs(Config) when is_list(Config) -> NVersion = ssl_test_lib:protocol_version(Config, tuple), - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:dsa_suites(NVersion), - ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), - run_suites(Ciphers, Version, Config, dsa). + run_suites(Ciphers, Config, dsa). %%------------------------------------------------------------------- ciphers_dsa_signed_certs_openssl_names() -> [{doc,"Test all dsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_dsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:openssl_dsa_suites(), - ct:log("tls1 openssl cipher suites ~p~n", [Ciphers]), - run_suites(Ciphers, Version, Config, dsa). + run_suites(Ciphers, Config, dsa). %%------------------------------------------------------------------- anonymous_cipher_suites()-> [{doc,"Test the anonymous ciphersuites"}]. anonymous_cipher_suites(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:anonymous_suites(Version), - run_suites(Ciphers, Version, Config, anonymous). + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:anonymous_suites(NVersion), + run_suites(Ciphers, Config, anonymous). %%------------------------------------------------------------------- psk_cipher_suites() -> [{doc, "Test the PSK ciphersuites WITHOUT server supplied identity hint"}]. psk_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = ssl_test_lib:protocol_version(Config), + NVersion = ssl_test_lib:protocol_version(Config, tuple), Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Version, Config, psk). + run_suites(Ciphers, Config, psk). %%------------------------------------------------------------------- psk_with_hint_cipher_suites()-> [{doc, "Test the PSK ciphersuites WITH server supplied identity hint"}]. psk_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = ssl_test_lib:protocol_version(Config), + NVersion = ssl_test_lib:protocol_version(Config, tuple), Ciphers = ssl_test_lib:psk_suites(NVersion), - run_suites(Ciphers, Version, Config, psk_with_hint). + run_suites(Ciphers, Config, psk_with_hint). %%------------------------------------------------------------------- psk_anon_cipher_suites() -> [{doc, "Test the anonymous PSK ciphersuites WITHOUT server supplied identity hint"}]. psk_anon_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = ssl_test_lib:protocol_version(Config), + NVersion = ssl_test_lib:protocol_version(Config, tuple), Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Version, Config, psk_anon). + run_suites(Ciphers, Config, psk_anon). %%------------------------------------------------------------------- psk_anon_with_hint_cipher_suites()-> [{doc, "Test the anonymous PSK ciphersuites WITH server supplied identity hint"}]. psk_anon_with_hint_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = ssl_test_lib:protocol_version(Config), + NVersion = ssl_test_lib:protocol_version(Config, tuple), Ciphers = ssl_test_lib:psk_anon_suites(NVersion), - run_suites(Ciphers, Version, Config, psk_anon_with_hint). + run_suites(Ciphers, Config, psk_anon_with_hint). %%------------------------------------------------------------------- srp_cipher_suites()-> [{doc, "Test the SRP ciphersuites"}]. srp_cipher_suites(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:srp_suites(), - run_suites(Ciphers, Version, Config, srp). + run_suites(Ciphers, Config, srp). %%------------------------------------------------------------------- srp_anon_cipher_suites()-> [{doc, "Test the anonymous SRP ciphersuites"}]. srp_anon_cipher_suites(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:srp_anon_suites(), - run_suites(Ciphers, Version, Config, srp_anon). + run_suites(Ciphers, Config, srp_anon). %%------------------------------------------------------------------- srp_dsa_cipher_suites()-> [{doc, "Test the SRP DSA ciphersuites"}]. srp_dsa_cipher_suites(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:srp_dss_suites(), - run_suites(Ciphers, Version, Config, srp_dsa). + run_suites(Ciphers, Config, srp_dsa). %%------------------------------------------------------------------- rc4_rsa_cipher_suites()-> [{doc, "Test the RC4 ciphersuites"}]. rc4_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = tls_record:protocol_version(NVersion), - Ciphers = ssl_test_lib:rc4_suites(NVersion), - run_suites(Ciphers, Version, Config, rc4_rsa). + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = [S || {rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], + run_suites(Ciphers, Config, rc4_rsa). %------------------------------------------------------------------- rc4_ecdh_rsa_cipher_suites()-> [{doc, "Test the RC4 ciphersuites"}]. rc4_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> - NVersion = tls_record:highest_protocol_version([]), - Version = tls_record:protocol_version(NVersion), - Ciphers = ssl_test_lib:rc4_suites(NVersion), - run_suites(Ciphers, Version, Config, rc4_ecdh_rsa). + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = [S || {ecdh_rsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], + run_suites(Ciphers, Config, rc4_ecdh_rsa). %%------------------------------------------------------------------- rc4_ecdsa_cipher_suites()-> [{doc, "Test the RC4 ciphersuites"}]. rc4_ecdsa_cipher_suites(Config) when is_list(Config) -> NVersion = tls_record:highest_protocol_version([]), - Version = tls_record:protocol_version(NVersion), - Ciphers = ssl_test_lib:rc4_suites(NVersion), - run_suites(Ciphers, Version, Config, rc4_ecdsa). + Ciphers = [S || {ecdhe_ecdsa,_,_} = S <- ssl_test_lib:rc4_suites(NVersion)], + run_suites(Ciphers, Config, rc4_ecdsa). %%------------------------------------------------------------------- des_rsa_cipher_suites()-> [{doc, "Test the des_rsa ciphersuites"}]. des_rsa_cipher_suites(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:des_suites(Config), - run_suites(Ciphers, Version, Config, des_rsa). + run_suites(Ciphers, Config, des_rsa). %------------------------------------------------------------------- des_ecdh_rsa_cipher_suites()-> [{doc, "Test ECDH rsa signed ciphersuites"}]. des_ecdh_rsa_cipher_suites(Config) when is_list(Config) -> NVersion = ssl_test_lib:protocol_version(Config, tuple), - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:des_suites(NVersion), - run_suites(Ciphers, Version, Config, des_dhe_rsa). + run_suites(Ciphers, Config, des_dhe_rsa). %%-------------------------------------------------------------------- default_reject_anonymous()-> @@ -2476,38 +2472,30 @@ ciphers_ecdsa_signed_certs() -> ciphers_ecdsa_signed_certs(Config) when is_list(Config) -> NVersion = ssl_test_lib:protocol_version(Config, tuple), - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:ecdsa_suites(NVersion), - ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), - run_suites(Ciphers, Version, Config, ecdsa). + run_suites(Ciphers, Config, ecdsa). %%-------------------------------------------------------------------- ciphers_ecdsa_signed_certs_openssl_names() -> [{doc, "Test all ecdsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_ecdsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:openssl_ecdsa_suites(), - ct:log("tls1 openssl cipher suites ~p~n", [Ciphers]), - run_suites(Ciphers, Version, Config, ecdsa). + run_suites(Ciphers, Config, ecdsa). %%-------------------------------------------------------------------- ciphers_ecdh_rsa_signed_certs() -> [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_ecdh_rsa_signed_certs(Config) when is_list(Config) -> NVersion = ssl_test_lib:protocol_version(Config, tuple), - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:ecdh_rsa_suites(NVersion), - ct:log("~p erlang cipher suites ~p~n", [Version, Ciphers]), - run_suites(Ciphers, Version, Config, ecdh_rsa). + run_suites(Ciphers, Config, ecdh_rsa). %%-------------------------------------------------------------------- ciphers_ecdh_rsa_signed_certs_openssl_names() -> [{doc, "Test all ecdh_rsa ssl cipher suites in highest support ssl/tls version"}]. ciphers_ecdh_rsa_signed_certs_openssl_names(Config) when is_list(Config) -> - Version = ssl_test_lib:protocol_version(Config), Ciphers = ssl_test_lib:openssl_ecdh_rsa_suites(), - ct:log("tls1 openssl cipher suites ~p~n", [Ciphers]), - run_suites(Ciphers, Version, Config, ecdh_rsa). + run_suites(Ciphers, Config, ecdh_rsa). %%-------------------------------------------------------------------- reuse_session() -> [{doc,"Test reuse of sessions (short handshake)"}]. @@ -3024,37 +3012,6 @@ der_input_opts(Opts) -> {Cert, {Asn1Type, Key}, CaCerts, DHParams}. %%-------------------------------------------------------------------- -%% different_ca_peer_sign() -> -%% ["Check that a CA can have a different signature algorithm than the peer cert."]; - -%% different_ca_peer_sign(Config) when is_list(Config) -> -%% ClientOpts = ssl_test_lib:ssl_options(client_mix_opts, Config), -%% ServerOpts = ssl_test_lib:ssl_options(server_mix_verify_opts, Config), - -%% {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), -%% Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, -%% {from, self()}, -%% {mfa, {ssl_test_lib, send_recv_result_active_once, []}}, -%% {options, [{active, once}, -%% {verify, verify_peer} | ServerOpts]}]), -%% Port = ssl_test_lib:inet_port(Server), - -%% Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, -%% {host, Hostname}, -%% {from, self()}, -%% {mfa, {ssl_test_lib, -%% send_recv_result_active_once, -%% []}}, -%% {options, [{active, once}, -%% {verify, verify_peer} -%% | ClientOpts]}]), - -%% ssl_test_lib:check_result(Server, ok, Client, ok), -%% ssl_test_lib:close(Server), -%% ssl_test_lib:close(Client). - - -%%-------------------------------------------------------------------- no_reuses_session_server_restart_new_cert() -> [{doc,"Check that a session is not reused if the server is restarted with a new cert."}]. no_reuses_session_server_restart_new_cert(Config) when is_list(Config) -> @@ -3122,14 +3079,14 @@ no_reuses_session_server_restart_new_cert_file(Config) when is_list(Config) -> DsaServerOpts = ssl_test_lib:ssl_options(server_dsa_opts, Config), PrivDir = proplists:get_value(priv_dir, Config), - NewServerOpts = new_config(PrivDir, ServerOpts), + NewServerOpts0 = new_config(PrivDir, ServerOpts), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, session_info_result, []}}, - {options, NewServerOpts}]), + {options, NewServerOpts0}]), Port = ssl_test_lib:inet_port(Server), Client0 = ssl_test_lib:start_client([{node, ClientNode}, @@ -3150,13 +3107,13 @@ no_reuses_session_server_restart_new_cert_file(Config) when is_list(Config) -> ssl:clear_pem_cache(), - NewServerOpts = new_config(PrivDir, DsaServerOpts), + NewServerOpts1 = new_config(PrivDir, DsaServerOpts), Server1 = ssl_test_lib:start_server([{node, ServerNode}, {port, Port}, {from, self()}, {mfa, {ssl_test_lib, no_result, []}}, - {options, NewServerOpts}]), + {options, NewServerOpts1}]), Client1 = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, @@ -3807,8 +3764,10 @@ no_rizzo_rc4() -> no_rizzo_rc4(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), - Ciphers = [ssl_cipher:erl_suite_definition(Suite) || - Suite <- ssl_test_lib:rc4_suites(tls_record:protocol_version(Version))], + NVersion = ssl_test_lib:protocol_version(Config, tuple), + %% Test uses RSA certs + Ciphers = ssl_test_lib:rc4_suites(NVersion) -- [{ecdhe_ecdsa,rc4_128,sha}, + {ecdh_ecdsa,rc4_128,sha}], run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -3818,7 +3777,8 @@ rizzo_one_n_minus_one() -> rizzo_one_n_minus_one(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), - AllSuites = ssl_test_lib:available_suites(tls_record:protocol_version(Version)), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + AllSuites = ssl_test_lib:available_suites(NVersion), Ciphers = [X || X ={_,Y,_} <- AllSuites, Y =/= rc4_128], run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_rizzo, []}). @@ -3829,7 +3789,8 @@ rizzo_zero_n() -> rizzo_zero_n(Config) when is_list(Config) -> Prop = proplists:get_value(tc_group_properties, Config), Version = proplists:get_value(name, Prop), - AllSuites = ssl_test_lib:available_suites(tls_record:protocol_version(Version)), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + AllSuites = ssl_test_lib:available_suites(NVersion), Ciphers = [X || X ={_,Y,_} <- AllSuites, Y =/= rc4_128], run_send_recv_rizzo(Ciphers, Config, Version, {?MODULE, send_recv_result_active_no_rizzo, []}). @@ -4631,7 +4592,10 @@ client_server_opts({KeyAlgo,_,_}, Config) when KeyAlgo == ecdh_rsa -> {ssl_test_lib:ssl_options(client_opts, Config), ssl_test_lib:ssl_options(server_ecdh_rsa_opts, Config)}. -run_suites(Ciphers, Version, Config, Type) -> +run_suites(Ciphers, Config, Type) -> + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Version = ssl_test_lib:protocol_version(Config), + ct:log("Running cipher suites ~p~n", [Ciphers]), {ClientOpts, ServerOpts} = case Type of rsa -> @@ -4643,23 +4607,24 @@ run_suites(Ciphers, Version, Config, Type) -> anonymous -> %% No certs in opts! {ssl_test_lib:ssl_options(client_verification_opts, Config), - [{reuseaddr, true}, {ciphers, ssl_test_lib:anonymous_suites(Version)}]}; + [{reuseaddr, true}, {ciphers, ssl_test_lib:anonymous_suites(NVersion)} | + ssl_test_lib:ssl_options([], Config)]}; psk -> {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, ssl_test_lib:psk_suites(Version)} | + [{ciphers, ssl_test_lib:psk_suites(NVersion)} | ssl_test_lib:ssl_options(server_psk, Config)]}; psk_with_hint -> {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, ssl_test_lib:psk_suites(Version)} | + [{ciphers, ssl_test_lib:psk_suites(NVersion)} | ssl_test_lib:ssl_options(server_psk_hint, Config) ]}; psk_anon -> {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, ssl_test_lib:psk_anon_suites(Version)} | + [{ciphers, ssl_test_lib:psk_anon_suites(NVersion)} | ssl_test_lib:ssl_options(server_psk_anon, Config)]}; psk_anon_with_hint -> {ssl_test_lib:ssl_options(client_psk, Config), - [{ciphers, ssl_test_lib:psk_anon_suites(Version)} | + [{ciphers, ssl_test_lib:psk_anon_suites(NVersion)} | ssl_test_lib:ssl_options(server_psk_anon_hint, Config)]}; srp -> {ssl_test_lib:ssl_options(client_srp, Config), diff --git a/lib/ssl/test/ssl_crl_SUITE.erl b/lib/ssl/test/ssl_crl_SUITE.erl index e293d183f7..668c76e38d 100644 --- a/lib/ssl/test/ssl_crl_SUITE.erl +++ b/lib/ssl/test/ssl_crl_SUITE.erl @@ -155,9 +155,15 @@ init_per_testcase(Case, Config0) -> DataDir = proplists:get_value(data_dir, Config), CertDir = filename:join(proplists:get_value(priv_dir, Config0), idp_crl), {CertOpts, Config} = init_certs(CertDir, idp_crl, Config), - {ok, _} = make_certs:all(DataDir, CertDir, CertOpts), - ct:timetrap({seconds, 6}), - [{cert_dir, CertDir} | Config]; + case make_certs:all(DataDir, CertDir, CertOpts) of + {ok, _} -> + ct:timetrap({seconds, 6}), + [{cert_dir, CertDir} | Config]; + _ -> + end_per_testcase(Case, Config0), + ssl_test_lib:clean_start(), + {skip, "Unable to create IDP crls"} + end; false -> end_per_testcase(Case, Config0), ssl_test_lib:clean_start(), diff --git a/lib/ssl/test/ssl_npn_handshake_SUITE.erl b/lib/ssl/test/ssl_npn_handshake_SUITE.erl index a02881f1ae..6bf2aa2786 100644 --- a/lib/ssl/test/ssl_npn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_npn_handshake_SUITE.erl @@ -95,8 +95,13 @@ init_per_group(GroupName, Config) -> Config end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. init_per_testcase(_TestCase, Config) -> ssl_test_lib:ct_log_supported_protocol_versions(Config), diff --git a/lib/ssl/test/ssl_packet_SUITE.erl b/lib/ssl/test/ssl_packet_SUITE.erl index 7281425461..408d62ce9c 100644 --- a/lib/ssl/test/ssl_packet_SUITE.erl +++ b/lib/ssl/test/ssl_packet_SUITE.erl @@ -53,28 +53,34 @@ all() -> {group, 'tlsv1.2'}, {group, 'tlsv1.1'}, {group, 'tlsv1'}, - {group, 'sslv3'} + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} ]. groups() -> - [{'tlsv1.2', [], packet_tests()}, - {'tlsv1.1', [], packet_tests()}, - {'tlsv1', [], packet_tests()}, - {'sslv3', [], packet_tests()} + [{'tlsv1.2', [], socket_packet_tests() ++ protocol_packet_tests()}, + {'tlsv1.1', [], socket_packet_tests() ++ protocol_packet_tests()}, + {'tlsv1', [], socket_packet_tests() ++ protocol_packet_tests()}, + {'sslv3', [], socket_packet_tests() ++ protocol_packet_tests()}, + {'dtlsv1.2', [], protocol_packet_tests()}, + {'dtlsv1', [], protocol_packet_tests()} ]. -packet_tests() -> - active_packet_tests() ++ active_once_packet_tests() ++ passive_packet_tests() ++ - [packet_send_to_large, - packet_cdr_decode, packet_cdr_decode_list, +socket_packet_tests() -> + socket_active_packet_tests() ++ socket_active_once_packet_tests() ++ + socket_passive_packet_tests() ++ [packet_send_to_large, packet_tpkt_decode, packet_tpkt_decode_list]. + +protocol_packet_tests() -> + protocol_active_packet_tests() ++ protocol_active_once_packet_tests() ++ protocol_passive_packet_tests() ++ + [packet_cdr_decode, packet_cdr_decode_list, packet_http_decode, packet_http_decode_list, packet_http_bin_decode_multi, packet_line_decode, packet_line_decode_list, packet_asn1_decode, packet_asn1_decode_list, - packet_tpkt_decode, packet_tpkt_decode_list, packet_sunrm_decode, packet_sunrm_decode_list]. -passive_packet_tests() -> +socket_passive_packet_tests() -> [packet_raw_passive_many_small, packet_0_passive_many_small, packet_1_passive_many_small, @@ -85,12 +91,8 @@ passive_packet_tests() -> packet_1_passive_some_big, packet_2_passive_some_big, packet_4_passive_some_big, - packet_httph_passive, - packet_httph_bin_passive, - packet_http_error_passive, packet_wait_passive, packet_size_passive, - packet_baddata_passive, %% inet header option should be deprecated! header_decode_one_byte_passive, header_decode_two_bytes_passive, @@ -98,7 +100,14 @@ passive_packet_tests() -> header_decode_two_bytes_one_sent_passive ]. -active_once_packet_tests() -> +protocol_passive_packet_tests() -> + [packet_httph_passive, + packet_httph_bin_passive, + packet_http_error_passive, + packet_baddata_passive + ]. + +socket_active_once_packet_tests() -> [packet_raw_active_once_many_small, packet_0_active_once_many_small, packet_1_active_once_many_small, @@ -108,12 +117,16 @@ active_once_packet_tests() -> packet_0_active_once_some_big, packet_1_active_once_some_big, packet_2_active_once_some_big, - packet_4_active_once_some_big, + packet_4_active_once_some_big + ]. + +protocol_active_once_packet_tests() -> + [ packet_httph_active_once, packet_httph_bin_active_once ]. -active_packet_tests() -> +socket_active_packet_tests() -> [packet_raw_active_many_small, packet_0_active_many_small, packet_1_active_many_small, @@ -124,10 +137,7 @@ active_packet_tests() -> packet_1_active_some_big, packet_2_active_some_big, packet_4_active_some_big, - packet_httph_active, - packet_httph_bin_active, packet_wait_active, - packet_baddata_active, packet_size_active, %% inet header option should be deprecated! header_decode_one_byte_active, @@ -136,6 +146,13 @@ active_packet_tests() -> header_decode_two_bytes_one_sent_active ]. + +protocol_active_packet_tests() -> + [packet_httph_active, + packet_httph_bin_active, + packet_baddata_active + ]. + init_per_suite(Config) -> catch crypto:stop(), try crypto:start() of @@ -168,8 +185,13 @@ init_per_group(GroupName, Config) -> end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. init_per_testcase(_TestCase, Config) -> ct:timetrap({seconds, ?BASE_TIMEOUT_SECONDS}), diff --git a/lib/ssl/test/ssl_payload_SUITE.erl b/lib/ssl/test/ssl_payload_SUITE.erl index cb1957327a..ef05241759 100644 --- a/lib/ssl/test/ssl_payload_SUITE.erl +++ b/lib/ssl/test/ssl_payload_SUITE.erl @@ -95,8 +95,13 @@ init_per_group(GroupName, Config) -> Config end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. init_per_testcase(TestCase, Config) when TestCase == server_echos_passive_huge; TestCase == server_echos_active_once_huge; diff --git a/lib/ssl/test/ssl_sni_SUITE.erl b/lib/ssl/test/ssl_sni_SUITE.erl index 4e916a7f03..03676cb828 100644 --- a/lib/ssl/test/ssl_sni_SUITE.erl +++ b/lib/ssl/test/ssl_sni_SUITE.erl @@ -30,21 +30,50 @@ %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- -all() -> [no_sni_header, - sni_match, - sni_no_match, - no_sni_header_fun, - sni_match_fun, - sni_no_match_fun]. +all() -> + [{group, 'tlsv1.2'}, + {group, 'tlsv1.1'}, + {group, 'tlsv1'}, + {group, 'sslv3'}, + {group, 'dtlsv1.2'}, + {group, 'dtlsv1'} + ]. + +groups() -> + [ + {'tlsv1.2', [], sni_tests()}, + {'tlsv1.1', [], sni_tests()}, + {'tlsv1', [], sni_tests()}, + {'sslv3', [], sni_tests()}, + {'dtlsv1.2', [], sni_tests()}, + {'dtlsv1', [], sni_tests()} + ]. + +sni_tests() -> + [no_sni_header, + sni_match, + sni_no_match, + no_sni_header_fun, + sni_match_fun, + sni_no_match_fun]. init_per_suite(Config0) -> catch crypto:stop(), try crypto:start() of ok -> ssl_test_lib:clean_start(), - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config0), - proplists:get_value(priv_dir, Config0)), - ssl_test_lib:cert_options(Config0) + Config = ssl_test_lib:make_rsa_cert(Config0), + RsaOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + [{sni_server_opts, [{sni_hosts, [ + {"a.server", [ + {certfile, proplists:get_value(certfile, RsaOpts)}, + {keyfile, proplists:get_value(keyfile, RsaOpts)} + ]}, + {"b.server", [ + {certfile, proplists:get_value(certfile, RsaOpts)}, + {keyfile, proplists:get_value(keyfile, RsaOpts)} + ]} + ]}]} | Config] catch _:_ -> {skip, "Crypto did not start"} end. @@ -66,22 +95,22 @@ end_per_testcase(_TestCase, Config) -> %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- no_sni_header(Config) -> - run_handshake(Config, undefined, undefined, "server"). + run_handshake(Config, undefined, undefined, "server Peer cert"). no_sni_header_fun(Config) -> - run_sni_fun_handshake(Config, undefined, undefined, "server"). + run_sni_fun_handshake(Config, undefined, undefined, "server Peer cert"). sni_match(Config) -> - run_handshake(Config, "a.server", "a.server", "a.server"). + run_handshake(Config, "a.server", "a.server", "server Peer cert"). sni_match_fun(Config) -> - run_sni_fun_handshake(Config, "a.server", "a.server", "a.server"). + run_sni_fun_handshake(Config, "a.server", "a.server", "server Peer cert"). sni_no_match(Config) -> - run_handshake(Config, "c.server", undefined, "server"). + run_handshake(Config, "c.server", undefined, "server Peer cert"). sni_no_match_fun(Config) -> - run_sni_fun_handshake(Config, "c.server", undefined, "server"). + run_sni_fun_handshake(Config, "c.server", undefined, "server Peer cert"). %%-------------------------------------------------------------------- @@ -141,13 +170,13 @@ run_sni_fun_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), [{sni_hosts, ServerSNIConf}] = proplists:get_value(sni_server_opts, Config), SNIFun = fun(Domain) -> proplists:get_value(Domain, ServerSNIConf, undefined) end, - ServerOptions = proplists:get_value(server_opts, Config) ++ [{sni_fun, SNIFun}], + ServerOptions = ssl_test_lib:ssl_options(server_rsa_opts, Config) ++ [{sni_fun, SNIFun}], ClientOptions = case SNIHostname of undefined -> - proplists:get_value(client_opts, Config); + ssl_test_lib:ssl_options(client_rsa_opts, Config); _ -> - [{server_name_indication, SNIHostname}] ++ proplists:get_value(client_opts, Config) + [{server_name_indication, SNIHostname}] ++ ssl_test_lib:ssl_options(client_rsa_opts, Config) end, ct:log("Options: ~p", [[ServerOptions, ClientOptions]]), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -167,14 +196,14 @@ run_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, " "ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), - ServerOptions = proplists:get_value(sni_server_opts, Config) ++ proplists:get_value(server_opts, Config), + ServerOptions = proplists:get_value(sni_server_opts, Config) ++ ssl_test_lib:ssl_options(server_rsa_opts, Config), ClientOptions = - case SNIHostname of - undefined -> - proplists:get_value(client_opts, Config); - _ -> - [{server_name_indication, SNIHostname}] ++ proplists:get_value(client_opts, Config) - end, + case SNIHostname of + undefined -> + ssl_test_lib:ssl_options(client_rsa_opts, Config); + _ -> + [{server_name_indication, SNIHostname}] ++ ssl_test_lib:ssl_options(client_rsa_opts, Config) + end, ct:log("Options: ~p", [[ServerOptions, ClientOptions]]), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 77c21d9b57..17eadcb8e9 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -384,10 +384,6 @@ cert_options(Config) -> "badkey.pem"]), PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, - SNIServerACertFile = filename:join([proplists:get_value(priv_dir, Config), "a.server", "cert.pem"]), - SNIServerAKeyFile = filename:join([proplists:get_value(priv_dir, Config), "a.server", "key.pem"]), - SNIServerBCertFile = filename:join([proplists:get_value(priv_dir, Config), "b.server", "cert.pem"]), - SNIServerBKeyFile = filename:join([proplists:get_value(priv_dir, Config), "b.server", "key.pem"]), [{client_opts, [{cacertfile, ClientCaCertFile}, {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]}, @@ -445,46 +441,34 @@ cert_options(Config) -> {server_bad_cert, [{ssl_imp, new},{cacertfile, ServerCaCertFile}, {certfile, BadCertFile}, {keyfile, ServerKeyFile}]}, {server_bad_key, [{ssl_imp, new},{cacertfile, ServerCaCertFile}, - {certfile, ServerCertFile}, {keyfile, BadKeyFile}]}, - {sni_server_opts, [{sni_hosts, [ - {"a.server", [ - {certfile, SNIServerACertFile}, - {keyfile, SNIServerAKeyFile} - ]}, - {"b.server", [ - {certfile, SNIServerBCertFile}, - {keyfile, SNIServerBKeyFile} - ]} - ]}]} + {certfile, ServerCertFile}, {keyfile, BadKeyFile}]} | Config]. -make_dsa_cert(Config) -> - {ServerCaCertFile, ServerCertFile, ServerKeyFile} = - make_cert_files("server", Config, dsa, dsa, "", []), - {ClientCaCertFile, ClientCertFile, ClientKeyFile} = - make_cert_files("client", Config, dsa, dsa, "", []), - [{server_dsa_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ServerCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}]}, - {server_dsa_verify_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ClientCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, - {verify, verify_peer}]}, - {client_dsa_opts, [{ssl_imp, new}, - {cacertfile, ClientCaCertFile}, - {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]}, - {server_srp_dsa, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ServerCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, - {user_lookup_fun, {fun user_lookup/3, undefined}}, - {ciphers, srp_dss_suites()}]}, - {client_srp_dsa, [{ssl_imp, new}, - {srp_identity, {"Test-User", "secret"}}, - {cacertfile, ClientCaCertFile}, - {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} - | Config]. - +make_dsa_cert(Config) -> + CryptoSupport = crypto:supports(), + case proplists:get_bool(dss, proplists:get_value(public_keys, CryptoSupport)) of + true -> + ClientFileBase = filename:join([proplists:get_value(priv_dir, Config), "dsa"]), + ServerFileBase = filename:join([proplists:get_value(priv_dir, Config), "dsa"]), + KeyGenSpec = key_gen_info(dsa, dsa), + + GenCertData = x509_test:gen_test_certs([{digest, sha} | KeyGenSpec]), + [{server_config, ServerConf}, + {client_config, ClientConf}] = + x509_test:gen_pem_config_files(GenCertData, ClientFileBase, ServerFileBase), + + [{server_dsa_opts, ServerConf}, + {server_dsa_verify_opts, [{verify, verify_peer} | ServerConf]}, + {client_dsa_opts, ClientConf}, + {server_srp_dsa, [{user_lookup_fun, {fun user_lookup/3, undefined}}, + {ciphers, srp_dss_suites()} | ServerConf]}, + {client_srp_dsa, [{srp_identity, {"Test-User", "secret"}} + | ClientConf]} + | Config]; + false -> + Config + end. make_rsa_cert_chains(ChainConf, Config, Suffix) -> CryptoSupport = crypto:supports(), KeyGenSpec = key_gen_info(rsa, rsa), @@ -541,6 +525,11 @@ key_gen_spec(Role, rsa) -> [{list_to_atom(Role ++ "_key_gen"), hardcode_rsa_key(1)}, {list_to_atom(Role ++ "_key_gen_chain"), [hardcode_rsa_key(2), hardcode_rsa_key(3)]} + ]; +key_gen_spec(Role, dsa) -> + [{list_to_atom(Role ++ "_key_gen"), hardcode_dsa_key(1)}, + {list_to_atom(Role ++ "_key_gen_chain"), [hardcode_dsa_key(2), + hardcode_dsa_key(3)]} ]. make_ecdsa_cert(Config) -> CryptoSupport = crypto:supports(), @@ -638,41 +627,6 @@ make_ecdh_rsa_cert(Config) -> Config end. -make_mix_cert(Config) -> - {ServerCaCertFile, ServerCertFile, ServerKeyFile} = make_cert_files("server", Config, dsa, - rsa, "mix", []), - {ClientCaCertFile, ClientCertFile, ClientKeyFile} = make_cert_files("client", Config, dsa, - rsa, "mix", []), - [{server_mix_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ServerCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}]}, - {server_mix_verify_opts, [{ssl_imp, new},{reuseaddr, true}, - {cacertfile, ClientCaCertFile}, - {certfile, ServerCertFile}, {keyfile, ServerKeyFile}, - {verify, verify_peer}]}, - {client_mix_opts, [{ssl_imp, new}, - {cacertfile, ClientCaCertFile}, - {certfile, ClientCertFile}, {keyfile, ClientKeyFile}]} - | Config]. - -make_cert_files(RoleStr, Config, Alg1, Alg2, Prefix, Opts) -> - Alg1Str = atom_to_list(Alg1), - Alg2Str = atom_to_list(Alg2), - CaInfo = {CaCert, _} = erl_make_certs:make_cert([{key, Alg1}| Opts]), - {Cert, CertKey} = erl_make_certs:make_cert([{key, Alg2}, {issuer, CaInfo} | Opts]), - CaCertFile = filename:join([proplists:get_value(priv_dir, Config), - RoleStr, Prefix ++ Alg1Str ++ "_cacerts.pem"]), - CertFile = filename:join([proplists:get_value(priv_dir, Config), - RoleStr, Prefix ++ Alg2Str ++ "_cert.pem"]), - KeyFile = filename:join([proplists:get_value(priv_dir, Config), - RoleStr, Prefix ++ Alg2Str ++ "_key.pem"]), - - der_to_pem(CaCertFile, [{'Certificate', CaCert, not_encrypted}]), - der_to_pem(CertFile, [{'Certificate', Cert, not_encrypted}]), - der_to_pem(KeyFile, [CertKey]), - {CaCertFile, CertFile, KeyFile}. - - start_upgrade_server(Args) -> Result = spawn_link(?MODULE, run_upgrade_server, [Args]), receive @@ -983,16 +937,10 @@ ecdh_rsa_suites(Version) -> end, available_suites(Version)). -openssl_rsa_suites(CounterPart) -> +openssl_rsa_suites() -> Ciphers = ssl:cipher_suites(openssl), - Names = case is_sane_ecc(CounterPart) of - true -> - "DSS | ECDSA"; - false -> - "DSS | ECDHE | ECDH" - end, - lists:filter(fun(Str) -> string_regex_filter(Str, Names) - end, Ciphers). + lists:filter(fun(Str) -> string_regex_filter(Str, "RSA") + end, Ciphers) -- openssl_ecdh_rsa_suites(). openssl_dsa_suites() -> Ciphers = ssl:cipher_suites(openssl), @@ -1026,11 +974,11 @@ string_regex_filter(_Str, _Search) -> false. anonymous_suites(Version) -> - Suites = ssl_cipher:anonymous_suites(Version), + Suites = [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:anonymous_suites(Version)], ssl_cipher:filter_suites(Suites). psk_suites(Version) -> - Suites = ssl_cipher:psk_suites(Version), + Suites = [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:psk_suites(Version)], ssl_cipher:filter_suites(Suites). psk_anon_suites(Version) -> @@ -1062,7 +1010,7 @@ srp_dss_suites() -> ssl_cipher:filter_suites(Suites). rc4_suites(Version) -> - Suites = ssl_cipher:rc4_suites(Version), + Suites = [ssl_cipher:erl_suite_definition(S) || S <- ssl_cipher:rc4_suites(Version)], ssl_cipher:filter_suites(Suites). des_suites(Version) -> @@ -1167,6 +1115,9 @@ init_tls_version(Version, Config) -> NewConfig = proplists:delete(protocol_opts, proplists:delete(protocol, Config)), [{protocol, tls} | NewConfig]. +clean_tls_version(Config) -> + proplists:delete(protocol_opts, proplists:delete(protocol, Config)). + sufficient_crypto_support(Version) when Version == 'tlsv1.2'; Version == 'dtlsv1.2' -> CryptoSupport = crypto:supports(), @@ -1276,7 +1227,7 @@ is_fips(_) -> false. cipher_restriction(Config0) -> - Version = tls_record:protocol_version(protocol_version(Config0)), + Version = protocol_version(Config0, tuple), case is_sane_ecc(openssl) of false -> Opts = proplists:get_value(server_opts, Config0), @@ -1294,6 +1245,8 @@ check_sane_openssl_version(Version) -> case supports_ssl_tls_version(Version) of true -> case {Version, os:cmd("openssl version")} of + {'sslv3', "OpenSSL 1.0.2" ++ _} -> + false; {_, "OpenSSL 1.0.2" ++ _} -> true; {_, "OpenSSL 1.0.1" ++ _} -> @@ -1365,6 +1318,12 @@ version_flag('dtlsv1.2') -> version_flag('dtlsv1') -> "-dtls1". +filter_suites([Cipher | _] = Ciphers, AtomVersion) when is_list(Cipher)-> + filter_suites([ssl_cipher:openssl_suite(S) || S <- Ciphers], + AtomVersion); +filter_suites([Cipher | _] = Ciphers, AtomVersion) when is_binary(Cipher)-> + filter_suites([ssl_cipher:erl_suite_definition(S) || S <- Ciphers], + AtomVersion); filter_suites(Ciphers0, AtomVersion) -> Version = tls_version(AtomVersion), Supported0 = ssl_cipher:suites(Version) @@ -1419,12 +1378,15 @@ supports_ssl_tls_version(sslv2 = Version) -> case os:cmd("openssl version") of "OpenSSL 1" ++ _ -> false; + %% Appears to be broken + "OpenSSL 0.9.8.o" ++ _ -> + false; _ -> VersionFlag = version_flag(Version), Exe = "openssl", Args = ["s_client", VersionFlag], Port = ssl_test_lib:portable_open_port(Exe, Args), - do_supports_ssl_tls_version(Port) + do_supports_ssl_tls_version(Port, "") end; supports_ssl_tls_version(Version) -> @@ -1432,23 +1394,26 @@ supports_ssl_tls_version(Version) -> Exe = "openssl", Args = ["s_client", VersionFlag], Port = ssl_test_lib:portable_open_port(Exe, Args), - do_supports_ssl_tls_version(Port). + do_supports_ssl_tls_version(Port, ""). -do_supports_ssl_tls_version(Port) -> +do_supports_ssl_tls_version(Port, Acc) -> receive - {Port, {data, "u"}} -> - false; - {Port, {data, "unknown option" ++ _}} -> - false; - {Port, {data, Data}} -> - case lists:member("error", string:tokens(Data, ":")) of - true -> - false; - false -> - do_supports_ssl_tls_version(Port) - end + {Port, {data, Data}} -> + case Acc ++ Data of + "unknown option" ++ _ -> + false; + Error when length(Error) >= 11 -> + case lists:member("error", string:tokens(Data, ":")) of + true -> + false; + false -> + do_supports_ssl_tls_version(Port, Error) + end; + _ -> + do_supports_ssl_tls_version(Port, Acc ++ Data) + end after 1000 -> - true + true end. ssl_options(Option, Config) when is_atom(Option) -> @@ -1493,6 +1458,7 @@ ct_log_supported_protocol_versions(Config) -> clean_env() -> application:unset_env(ssl, protocol_version), + application:unset_env(ssl, dtls_protocol_version), application:unset_env(ssl, session_lifetime), application:unset_env(ssl, session_cb), application:unset_env(ssl, session_cb_init_args), @@ -1512,10 +1478,14 @@ is_psk_anon_suite({psk, _,_}) -> true; is_psk_anon_suite({dhe_psk,_,_}) -> true; +is_psk_anon_suite({ecdhe_psk,_,_}) -> + true; is_psk_anon_suite({psk, _,_,_}) -> true; is_psk_anon_suite({dhe_psk, _,_,_}) -> true; +is_psk_anon_suite({ecdhe_psk, _,_,_}) -> + true; is_psk_anon_suite(_) -> false. @@ -1535,7 +1505,7 @@ tls_version(Atom) -> tls_record:protocol_version(Atom). hardcode_rsa_key(1) -> - {'RSAPrivateKey',0, + {'RSAPrivateKey', 'two-prime', 23995666614853919027835084074500048897452890537492185072956789802729257783422306095699263934587064480357348855732149402060270996295002843755712064937715826848741191927820899197493902093529581182351132392364214171173881547273475904587683433713767834856230531387991145055273426806331200574039205571401702219159773947658558490957010003143162250693492642996408861265758000254664396313741422909188635443907373976005987612936763564996605457102336549804831742940035613780926178523017685712710473543251580072875247250504243621640157403744718833162626193206685233710319205099867303242759099560438381385658382486042995679707669, 17, 11292078406990079542510627799764728892919007311761028269626724613049062486316379339152594792746853873109340637991599718616598115903530750002688030558925094987642913848386305504703012749896273497577003478759630198199473669305165131570674557041773098755873191241407597673069847908861741446606684974777271632545629600685952292605647052193819136445675100211504432575554351515262198132231537860917084269870590492135731720141577986787033006338680118008484613510063003323516659048210893001173583018220214626635609151105287049126443102976056146630518124476470236027123782297108342869049542023328584384300970694412006494684657, @@ -1547,7 +1517,7 @@ hardcode_rsa_key(1) -> asn1_NOVALUE}; hardcode_rsa_key(2) -> -{'RSAPrivateKey',0, +{'RSAPrivateKey', 'two-prime', 21343679768589700771839799834197557895311746244621307033143551583788179817796325695589283169969489517156931770973490560582341832744966317712674900833543896521418422508485833901274928542544381247956820115082240721897193055368570146764204557110415281995205343662628196075590438954399631753508888358737971039058298703003743872818150364935790613286541190842600031570570099801682794056444451081563070538409720109449780410837763602317050353477918147758267825417201591905091231778937606362076129350476690460157227101296599527319242747999737801698427160817755293383890373574621116766934110792127739174475029121017282777887777, 17, 18832658619343853622211588088997845201745658451136447382185486691577805721584993260814073385267196632785528033211903435807948675951440868570007265441362261636545666919252206383477878125774454042314841278013741813438699754736973658909592256273895837054592950290554290654932740253882028017801960316533503857992358685308186680144968293076156011747178275038098868263178095174694099811498968993700538293188879611375604635940554394589807673542938082281934965292051746326331046224291377703201248790910007232374006151098976879987912446997911775904329728563222485791845480864283470332826504617837402078265424772379987120023773, @@ -1559,7 +1529,7 @@ hardcode_rsa_key(2) -> asn1_NOVALUE}; hardcode_rsa_key(3) -> -{'RSAPrivateKey',0, +{'RSAPrivateKey', 'two-prime', 25089040456112869869472694987833070928503703615633809313972554887193090845137746668197820419383804666271752525807484521370419854590682661809972833718476098189250708650325307850184923546875260207894844301992963978994451844985784504212035958130279304082438876764367292331581532569155681984449177635856426023931875082020262146075451989132180409962870105455517050416234175675478291534563995772675388370042873175344937421148321291640477650173765084699931690748536036544188863178325887393475703801759010864779559318631816411493486934507417755306337476945299570726975433250753415110141783026008347194577506976486290259135429, 17, 8854955455098659953931539407470495621824836570223697404931489960185796768872145882893348383311931058684147950284994536954265831032005645344696294253579799360912014817761873358888796545955974191021709753644575521998041827642041589721895044045980930852625485916835514940558187965584358347452650930302268008446431977397918214293502821599497633970075862760001650736520566952260001423171553461362588848929781360590057040212831994258783694027013289053834376791974167294527043946669963760259975273650548116897900664646809242902841107022557239712438496384819445301703021164043324282687280801738470244471443835900160721870265, @@ -1570,7 +1540,7 @@ hardcode_rsa_key(3) -> 15068630434698373319269196003209754243798959461311186548759287649485250508074064775263867418602372588394608558985183294561315208336731894947137343239541687540387209051236354318837334154993136528453613256169847839789803932725339395739618592522865156272771578671216082079933457043120923342632744996962853951612, asn1_NOVALUE}; hardcode_rsa_key(4) -> -{'RSAPrivateKey',0, +{'RSAPrivateKey', 'two-prime', 28617237755030755643854803617273584643843067580642149032833640135949799721163782522787597288521902619948688786051081993247908700824196122780349730169173433743054172191054872553484065655968335396052034378669869864779940355219732200954630251223541048434478476115391643898092650304645086338265930608997389611376417609043761464100338332976874588396803891301015812818307951159858145399281035705713082131199940309445719678087542976246147777388465712394062188801177717719764254900022006288880246925156931391594131839991579403409541227225173269459173129377291869028712271737734702830877034334838181789916127814298794576266389, 17, 26933870828264240605980991639786903194205240075898493207372837775011576208154148256741268036255908348187001210401018346586267012540419880263858569570986761169933338532757527109161473558558433313931326474042230460969355628442100895016122589386862163232450330461545076609969553227901257730132640573174013751883368376011370428995523268034111482031427024082719896108094847702954695363285832195666458915142143884210891427766607838346722974883433132513540317964796373298134261669479023445911856492129270184781873446960437310543998533283339488055776892320162032014809906169940882070478200435536171854883284366514852906334641, @@ -1581,7 +1551,7 @@ hardcode_rsa_key(4) -> 34340318160575773065401929915821192439103777558577109939078671096408836197675640654693301707202885840826672396546056002756167635035389371579540325327619480512374920136684787633921441576901246290213545161954865184290700344352088099063404416346968182170720521708773285279884132629954461545103181082503707725012, asn1_NOVALUE}; hardcode_rsa_key(5) -> -{'RSAPrivateKey',0, +{'RSAPrivateKey', 'two-prime', 26363170152814518327068346871197765236382539835597898797762992537312221863402655353436079974302838986536256364057947538018476963115004626096654613827403121905035011992899481598437933532388248462251770039307078647864188314916665766359828262009578648593031111569685489178543405615478739906285223620987558499488359880003693226535420421293716164794046859453204135383236667988765227190694994861629971618548127529849059769249520775574008363789050621665120207265361610436965088511042779948238320901918522125988916609088415989475825860046571847719492980547438560049874493788767083330042728150253120940100665370844282489982633, 17, 10855423004100095781734025182257903332628104638187370093196526338893267826106975733767797636477639582691399679317978398007608161282648963686857782164224814902073240232370374775827384395689278778574258251479385325591136364965685903795223402003944149420659869469870495544106108194608892902588033255700759382142132115013969680562678811046675523365751498355532768935784747314021422035957153013494814430893022253205880275287307995039363642554998244274484818208792520243113824379110193356010059999642946040953102866271737127640405568982049887176990990501963784502429481034227543991366980671390566584211881030995602076468001, @@ -1592,7 +1562,7 @@ hardcode_rsa_key(5) -> 40624877259097915043489529504071755460170951428490878553842519165800720914888257733191322215286203357356050737713125202129282154441426952501134581314792133018830748896123382106683994268028624341502298766844710276939303555637478596035491641473828661569958212421472263269629366559343208764012473880251174832392, asn1_NOVALUE}; hardcode_rsa_key(6) -> -{'RSAPrivateKey',0, +{'RSAPrivateKey', 'two-prime', 22748888494866396715768692484866595111939200209856056370972713870125588774286266397044592487895293134537316190976192161177144143633669641697309689280475257429554879273045671863645233402796222694405634510241820106743648116753479926387434021380537483429927516962909367257212902212159798399531316965145618774905828756510318897899298783143203190245236381440043169622358239226123652592179006905016804587837199618842875361941208299410035232803124113612082221121192550063791073372276763648926636149384299189072950588522522800393261949880796214514243704858378436010975184294077063518776479282353562934591448646412389762167039, 17, 6690849557313646092873144848490175032923294179369428344403739373566349639495960705013115437616262686628622409110644753287395336362844012263914614494257428655751435080307550548130951000822418439531068973600535325512837681398082331290421770994275730420566916753796872722709677121223470117509210872101652580854566448661533030419787125312956120661097410038933324613372774190658239039998357548275441758790939430824924502690997433186652165055694361752689819209062683281242276039100201318203707142383491769671330743466041394101421674581185260900666085723130684175548215193875544802254923825103844262661010117443222587769713, @@ -1603,6 +1573,27 @@ hardcode_rsa_key(6) -> 81173034184183681160439870161505779100040258708276674532866007896310418779840630960490793104541748007902477778658270784073595697910785917474138815202903114440800310078464142273778315781957021015333260021813037604142367434117205299831740956310682461174553260184078272196958146289378701001596552915990080834227, asn1_NOVALUE}. +hardcode_dsa_key(1) -> + {'DSAPrivateKey',0, + 99438313664986922963487511141216248076486724382260996073922424025828494981416579966171753999204426907349400798052572573634137057487829150578821328280864500098312146772602202702021153757550650696224643730869835650674962433068943942837519621267815961566259265204876799778977478160416743037274938277357237615491, + 1454908511695148818053325447108751926908854531909, + 20302424198893709525243209250470907105157816851043773596964076323184805650258390738340248469444700378962907756890306095615785481696522324901068493502141775433048117442554163252381401915027666416630898618301033737438756165023568220631119672502120011809327566543827706483229480417066316015458225612363927682579, + 48598545580251057979126570873881530215432219542526130654707948736559463436274835406081281466091739849794036308281564299754438126857606949027748889019480936572605967021944405048011118039171039273602705998112739400664375208228641666852589396502386172780433510070337359132965412405544709871654840859752776060358, + 1457508827177594730669011716588605181448418352823}; +hardcode_dsa_key(2) -> + {'DSAPrivateKey',0, + 145447354557382582722944332987784622105075065624518040072393858097520305927329240484963764783346271194321683798321743658303478090647837211867389721684646254999291098347011037298359107547264573476540026676832159205689428125157386525591130716464335426605521884822982379206842523670736739023467072341958074788151, + 742801637799670234315651916144768554943688916729, + 79727684678125120155622004643594683941478642656111969487719464672433839064387954070113655822700268007902716505761008423792735229036965034283173483862273639257533568978482104785033927768441235063983341565088899599358397638308472931049309161811156189887217888328371767967629005149630676763492409067382020352505, + 35853727034965131665219275925554159789667905059030049940938124723126925435403746979702929280654735557166864135215989313820464108440192507913554896358611966877432546584986661291483639036057475682547385322659469460385785257933737832719745145778223672383438466035853830832837226950912832515496378486927322864228, + 801315110178350279541885862867982846569980443911}; +hardcode_dsa_key(3) -> + {'DSAPrivateKey',0, + 99438313664986922963487511141216248076486724382260996073922424025828494981416579966171753999204426907349400798052572573634137057487829150578821328280864500098312146772602202702021153757550650696224643730869835650674962433068943942837519621267815961566259265204876799778977478160416743037274938277357237615491, + 1454908511695148818053325447108751926908854531909, + 20302424198893709525243209250470907105157816851043773596964076323184805650258390738340248469444700378962907756890306095615785481696522324901068493502141775433048117442554163252381401915027666416630898618301033737438756165023568220631119672502120011809327566543827706483229480417066316015458225612363927682579, + 48598545580251057979126570873881530215432219542526130654707948736559463436274835406081281466091739849794036308281564299754438126857606949027748889019480936572605967021944405048011118039171039273602705998112739400664375208228641666852589396502386172780433510070337359132965412405544709871654840859752776060358, + 1457508827177594730669011716588605181448418352823}. dtls_hello() -> [1, diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 5093ef3728..c4fe97d88e 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -85,19 +85,19 @@ all_versions_tests() -> ]. dtls_all_versions_tests() -> [ - %%erlang_client_openssl_server, + erlang_client_openssl_server, erlang_server_openssl_client, - %%erlang_client_openssl_server_dsa_cert, + erlang_client_openssl_server_dsa_cert, erlang_server_openssl_client_dsa_cert, - erlang_server_openssl_client_reuse_session + erlang_server_openssl_client_reuse_session, %%erlang_client_openssl_server_renegotiate, %%erlang_client_openssl_server_nowrap_seqnum, %%erlang_server_openssl_client_nowrap_seqnum, - %%erlang_client_openssl_server_no_server_ca_cert, - %%erlang_client_openssl_server_client_cert, - %%erlang_server_openssl_client_client_cert - %%ciphers_rsa_signed_certs, - %%ciphers_dsa_signed_certs, + erlang_client_openssl_server_no_server_ca_cert, + erlang_client_openssl_server_client_cert, + erlang_server_openssl_client_client_cert, + ciphers_rsa_signed_certs, + ciphers_dsa_signed_certs %%erlang_client_bad_openssl_server, %%expired_session ]. @@ -142,12 +142,11 @@ init_per_suite(Config0) -> catch crypto:stop(), try crypto:start() of ok -> - ssl_test_lib:clean_start(), - {ok, _} = make_certs:all(proplists:get_value(data_dir, Config0), - proplists:get_value(priv_dir, Config0)), - Config1 = ssl_test_lib:make_dsa_cert(Config0), - Config = ssl_test_lib:cert_options(Config1), - ssl_test_lib:cipher_restriction(Config) + ssl_test_lib:clean_start(), + + Config1 = ssl_test_lib:make_rsa_cert(Config0), + Config2 = ssl_test_lib:make_dsa_cert(Config1), + ssl_test_lib:cipher_restriction(Config2) catch _:_ -> {skip, "Crypto did not start"} end @@ -183,8 +182,13 @@ init_per_group(GroupName, Config) -> Config end. -end_per_group(_GroupName, Config) -> - Config. +end_per_group(GroupName, Config) -> + case ssl_test_lib:is_tls_version(GroupName) of + true -> + ssl_test_lib:clean_tls_version(Config); + false -> + Config + end. init_per_testcase(expired_session, Config) -> ct:timetrap(?EXPIRE * 1000 * 5), @@ -196,7 +200,7 @@ init_per_testcase(expired_session, Config) -> init_per_testcase(TestCase, Config) when TestCase == ciphers_rsa_signed_certs; TestCase == ciphers_dsa_signed_certs -> - ct:timetrap({seconds, 45}), + ct:timetrap({seconds, 60}), special_init(TestCase, Config); init_per_testcase(TestCase, Config) -> @@ -270,13 +274,24 @@ special_init(TestCase, Config) check_openssl_npn_support(Config) end; -special_init(TestCase, Config) +special_init(TestCase, Config0) when TestCase == erlang_server_openssl_client_sni_match; TestCase == erlang_server_openssl_client_sni_no_match; TestCase == erlang_server_openssl_client_sni_no_header; TestCase == erlang_server_openssl_client_sni_match_fun; TestCase == erlang_server_openssl_client_sni_no_match_fun; TestCase == erlang_server_openssl_client_sni_no_header_fun -> + RsaOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config0), + Config = [{sni_server_opts, [{sni_hosts, + [{"a.server", [ + {certfile, proplists:get_value(certfile, RsaOpts)}, + {keyfile, proplists:get_value(keyfile, RsaOpts)} + ]}, + {"b.server", [ + {certfile, proplists:get_value(certfile, RsaOpts)}, + {keyfile, proplists:get_value(keyfile, RsaOpts)} + ]} + ]}]} | Config0], check_openssl_sni_support(Config); special_init(_, Config) -> @@ -295,8 +310,8 @@ basic_erlang_client_openssl_server() -> [{doc,"Test erlang client with openssl server"}]. basic_erlang_client_openssl_server(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -335,10 +350,10 @@ basic_erlang_server_openssl_client() -> [{doc,"Test erlang server with openssl client"}]. basic_erlang_server_openssl_client(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), V2Compat = proplists:get_value(v2_hello_compatible, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", ct:pal("v2_hello_compatible: ~p", [V2Compat]), @@ -351,7 +366,8 @@ basic_erlang_server_openssl_client(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), Exe = "openssl", - Args = ["s_client", "-connect", "localhost:" ++ integer_to_list(Port) | workaround_openssl_s_clinent()], + Args = ["s_client", "-connect", hostname_format(Hostname) ++ + ":" ++ integer_to_list(Port) | workaround_openssl_s_clinent()], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), true = port_command(OpenSslPort, Data), @@ -368,8 +384,8 @@ erlang_client_openssl_server() -> [{doc,"Test erlang client with openssl server"}]. erlang_client_openssl_server(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -408,9 +424,9 @@ erlang_server_openssl_client() -> [{doc,"Test erlang server with openssl client"}]. erlang_server_openssl_client(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", @@ -422,7 +438,7 @@ erlang_server_openssl_client(Config) when is_list(Config) -> Version = ssl_test_lib:protocol_version(Config), Exe = "openssl", - Args = ["s_client", "-connect", "localhost: " ++ integer_to_list(Port), + Args = ["s_client", "-connect", hostname_format(Hostname) ++":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version)], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), @@ -441,7 +457,7 @@ erlang_client_openssl_server_dsa_cert() -> erlang_client_openssl_server_dsa_cert(Config) when is_list(Config) -> process_flag(trap_exit, true), ClientOpts = ssl_test_lib:ssl_options(client_dsa_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_dsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_dsa_verify_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -486,7 +502,7 @@ erlang_server_openssl_client_dsa_cert(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_dsa_opts, Config), ServerOpts = ssl_test_lib:ssl_options(server_dsa_verify_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", CaCertFile = proplists:get_value(cacertfile, ClientOpts), @@ -500,7 +516,7 @@ erlang_server_openssl_client_dsa_cert(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), Version = ssl_test_lib:protocol_version(Config), Exe = "openssl", - Args = ["s_client", "-connect", "localhost: " ++ integer_to_list(Port), + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version), "-cert", CertFile, "-CAfile", CaCertFile, @@ -523,9 +539,9 @@ erlang_server_openssl_client_reuse_session() -> "same session id, to test reusing of sessions."}]. erlang_server_openssl_client_reuse_session(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", @@ -538,7 +554,8 @@ erlang_server_openssl_client_reuse_session(Config) when is_list(Config) -> Version = ssl_test_lib:protocol_version(Config), Exe = "openssl", - Args = ["s_client", "-connect", "localhost:" ++ integer_to_list(Port), + Args = ["s_client", "-connect", hostname_format(Hostname) + ++ ":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version), "-reconnect"], @@ -560,8 +577,8 @@ erlang_client_openssl_server_renegotiate() -> [{doc,"Test erlang client when openssl server issuses a renegotiate"}]. erlang_client_openssl_server_renegotiate(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -610,8 +627,8 @@ erlang_client_openssl_server_nowrap_seqnum() -> " to lower treashold substantially."}]. erlang_client_openssl_server_nowrap_seqnum(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -653,9 +670,9 @@ erlang_server_openssl_client_nowrap_seqnum() -> " to lower treashold substantially."}]. erlang_server_openssl_client_nowrap_seqnum(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", @@ -669,7 +686,7 @@ erlang_server_openssl_client_nowrap_seqnum(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), Version = ssl_test_lib:protocol_version(Config), Exe = "openssl", - Args = ["s_client","-connect", "localhost: " ++ integer_to_list(Port), + Args = ["s_client","-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version), "-msg"], @@ -692,8 +709,8 @@ erlang_client_openssl_server_no_server_ca_cert() -> "implicitly tested eleswhere."}]. erlang_client_openssl_server_no_server_ca_cert(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -733,8 +750,8 @@ erlang_client_openssl_server_client_cert() -> [{doc,"Test erlang client with openssl server when client sends cert"}]. erlang_client_openssl_server_client_cert(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -776,10 +793,10 @@ erlang_server_openssl_client_client_cert() -> [{doc,"Test erlang server with openssl client when client sends cert"}]. erlang_server_openssl_client_client_cert(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_verify_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_verify_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", @@ -799,7 +816,7 @@ erlang_server_openssl_client_client_cert(Config) when is_list(Config) -> Exe = "openssl", Args = ["s_client", "-cert", CertFile, "-CAfile", CaCertFile, - "-key", KeyFile,"-connect", "localhost:" ++ integer_to_list(Port), + "-key", KeyFile,"-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version)], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), @@ -817,8 +834,8 @@ erlang_server_erlang_client_client_cert() -> [{doc,"Test erlang server with erlang client when client sends cert"}]. erlang_server_erlang_client_client_cert(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = proplists:get_value(server_verification_opts, Config), - ClientOpts = proplists:get_value(client_verification_opts, Config), + ServerOpts = proplists:get_value(server_rsa_verify_opts, Config), + ClientOpts = proplists:get_value(client_rsa_verify_opts, Config), Version = ssl_test_lib:protocol_version(Config), {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), @@ -863,7 +880,8 @@ ciphers_dsa_signed_certs() -> [{doc,"Test cipher suites that uses dsa certs"}]. ciphers_dsa_signed_certs(Config) when is_list(Config) -> Version = ssl_test_lib:protocol_version(Config), - Ciphers = ssl_test_lib:dsa_suites(tls_record:protocol_version(Version)), + NVersion = ssl_test_lib:protocol_version(Config, tuple), + Ciphers = ssl_test_lib:dsa_suites(NVersion), run_suites(Ciphers, Version, Config, dsa). %%-------------------------------------------------------------------- @@ -871,8 +889,8 @@ erlang_client_bad_openssl_server() -> [{doc,"Test what happens if openssl server sends garbage to erlang ssl client"}]. erlang_client_bad_openssl_server(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_verification_opts, Config), - ClientOpts = ssl_test_lib:ssl_options(client_verification_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -926,8 +944,8 @@ expired_session() -> "better code coverage of the ssl_manager module"}]. expired_session(Config) when is_list(Config) -> process_flag(trap_exit, true), - ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ClientOpts = ssl_test_lib:ssl_options(client_rsa_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), Port = ssl_test_lib:inet_port(node()), @@ -980,9 +998,9 @@ ssl2_erlang_server_openssl_client() -> ssl2_erlang_server_openssl_client(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0}, {from, self()}, @@ -990,7 +1008,7 @@ ssl2_erlang_server_openssl_client(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), Exe = "openssl", - Args = ["s_client", "-connect", "localhost:" ++ integer_to_list(Port), + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), "-ssl2", "-msg"], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), @@ -1005,12 +1023,12 @@ ssl2_erlang_server_openssl_client_comp() -> ssl2_erlang_server_openssl_client_comp(Config) when is_list(Config) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), V2Compat = proplists:get_value(v2_hello_compatible, Config), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Data = "From openssl to erlang", @@ -1020,7 +1038,7 @@ ssl2_erlang_server_openssl_client_comp(Config) when is_list(Config) -> Port = ssl_test_lib:inet_port(Server), Exe = "openssl", - Args = ["s_client", "-connect", "localhost:" ++ integer_to_list(Port), + Args = ["s_client", "-connect", hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), "-ssl2", "-msg"], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), @@ -1248,22 +1266,22 @@ erlang_server_openssl_client_npn_only_client(Config) when is_list(Config) -> ok. %-------------------------------------------------------------------------- erlang_server_openssl_client_sni_no_header(Config) when is_list(Config) -> - erlang_server_openssl_client_sni_test(Config, undefined, undefined, "server"). + erlang_server_openssl_client_sni_test(Config, undefined, undefined, "server Peer cert"). erlang_server_openssl_client_sni_no_header_fun(Config) when is_list(Config) -> - erlang_server_openssl_client_sni_test_sni_fun(Config, undefined, undefined, "server"). + erlang_server_openssl_client_sni_test_sni_fun(Config, undefined, undefined, "server Peer cert"). -erlang_server_openssl_client_sni_match(Config) when is_list(Config) -> - erlang_server_openssl_client_sni_test(Config, "a.server", "a.server", "a.server"). +erlang_server_openssl_client_sni_match(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test(Config, "a.server", "a.server", "server Peer cert"). erlang_server_openssl_client_sni_match_fun(Config) when is_list(Config) -> - erlang_server_openssl_client_sni_test_sni_fun(Config, "a.server", "a.server", "a.server"). + erlang_server_openssl_client_sni_test_sni_fun(Config, "a.server", "a.server", "server Peer cert"). erlang_server_openssl_client_sni_no_match(Config) when is_list(Config) -> - erlang_server_openssl_client_sni_test(Config, "c.server", undefined, "server"). + erlang_server_openssl_client_sni_test(Config, "c.server", undefined, "server Peer cert"). erlang_server_openssl_client_sni_no_match_fun(Config) when is_list(Config) -> - erlang_server_openssl_client_sni_test_sni_fun(Config, "c.server", undefined, "server"). + erlang_server_openssl_client_sni_test_sni_fun(Config, "c.server", undefined, "server Peer cert"). %%-------------------------------------------------------------------- @@ -1273,11 +1291,11 @@ run_suites(Ciphers, Version, Config, Type) -> {ClientOpts, ServerOpts} = case Type of rsa -> - {ssl_test_lib:ssl_options(client_opts, Config), - ssl_test_lib:ssl_options(server_opts, Config)}; + {ssl_test_lib:ssl_options(client_rsa_opts, Config), + ssl_test_lib:ssl_options(server_rsa_opts, Config)}; dsa -> - {ssl_test_lib:ssl_options(client_opts, Config), - ssl_test_lib:ssl_options(server_dsa_opts, Config)} + {ssl_test_lib:ssl_options(client_dsa_opts, Config), + ssl_test_lib:ssl_options(server_dsa_verify_opts, Config)} end, Result = lists:map(fun(Cipher) -> @@ -1330,7 +1348,7 @@ send_and_hostname(SSLSocket) -> erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), - ServerOptions = proplists:get_value(sni_server_opts, Config) ++ proplists:get_value(server_opts, Config), + ServerOptions = proplists:get_value(sni_server_opts, Config) ++ proplists:get_value(server_rsa_opts, Config), {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {?MODULE, send_and_hostname, []}}, @@ -1344,11 +1362,7 @@ erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, openssl_client_args(ssl_test_lib:supports_ssl_tls_version(sslv2), Hostname, Port, SNIHostname) end, ClientPort = ssl_test_lib:portable_open_port(Exe, ClientArgs), - - %% Client check needs to be done befor server check, - %% or server check might consume client messages - ExpectedClientOutput = ["OK", "/CN=" ++ ExpectedCN ++ "/"], - client_check_result(ClientPort, ExpectedClientOutput), + ssl_test_lib:check_result(Server, ExpectedSNIHostname), ssl_test_lib:close_port(ClientPort), ssl_test_lib:close(Server), @@ -1359,7 +1373,7 @@ erlang_server_openssl_client_sni_test_sni_fun(Config, SNIHostname, ExpectedSNIHo ct:log("Start running handshake for sni_fun, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), [{sni_hosts, ServerSNIConf}] = proplists:get_value(sni_server_opts, Config), SNIFun = fun(Domain) -> proplists:get_value(Domain, ServerSNIConf, undefined) end, - ServerOptions = proplists:get_value(server_opts, Config) ++ [{sni_fun, SNIFun}], + ServerOptions = proplists:get_value(server_rsa_opts, Config) ++ [{sni_fun, SNIFun}], {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {?MODULE, send_and_hostname, []}}, @@ -1375,10 +1389,6 @@ erlang_server_openssl_client_sni_test_sni_fun(Config, SNIHostname, ExpectedSNIHo ClientPort = ssl_test_lib:portable_open_port(Exe, ClientArgs), - %% Client check needs to be done befor server check, - %% or server check might consume client messages - ExpectedClientOutput = ["OK", "/CN=" ++ ExpectedCN ++ "/"], - client_check_result(ClientPort, ExpectedClientOutput), ssl_test_lib:check_result(Server, ExpectedSNIHostname), ssl_test_lib:close_port(ClientPort), ssl_test_lib:close(Server). @@ -1442,8 +1452,8 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> start_erlang_client_and_openssl_server_with_opts(Config, ErlangClientOpts, OpensslServerOpts, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts0 = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), ClientOpts = ErlangClientOpts ++ ClientOpts0, {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -1488,8 +1498,8 @@ start_erlang_client_and_openssl_server_with_opts(Config, ErlangClientOpts, Opens start_erlang_client_and_openssl_server_for_alpn_negotiation(Config, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts = proplists:get_value(server_opts, Config), - ClientOpts0 = proplists:get_value(client_opts, Config), + ServerOpts = proplists:get_value(server_rsa_opts, Config), + ClientOpts0 = proplists:get_value(client_rsa_opts, Config), ClientOpts = [{alpn_advertised_protocols, [<<"spdy/2">>]} | ClientOpts0], {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -1524,7 +1534,7 @@ start_erlang_client_and_openssl_server_for_alpn_negotiation(Config, Data, Callba start_erlang_server_and_openssl_client_for_alpn_negotiation(Config, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts0 = proplists:get_value(server_opts, Config), + ServerOpts0 = proplists:get_value(server_rsa_opts, Config), ServerOpts = [{alpn_preferred_protocols, [<<"spdy/2">>]} | ServerOpts0], {_, ServerNode, _} = ssl_test_lib:run_where(Config), @@ -1553,8 +1563,8 @@ start_erlang_server_and_openssl_client_for_alpn_negotiation(Config, Data, Callba start_erlang_client_and_openssl_server_for_alpn_npn_negotiation(Config, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts = proplists:get_value(server_opts, Config), - ClientOpts0 = proplists:get_value(client_opts, Config), + ServerOpts = proplists:get_value(server_rsa_opts, Config), + ClientOpts0 = proplists:get_value(client_rsa_opts, Config), ClientOpts = [{alpn_advertised_protocols, [<<"spdy/2">>]}, {client_preferred_next_protocols, {client, [<<"spdy/3">>, <<"http/1.1">>]}} | ClientOpts0], @@ -1593,7 +1603,7 @@ start_erlang_client_and_openssl_server_for_alpn_npn_negotiation(Config, Data, Ca start_erlang_server_and_openssl_client_for_alpn_npn_negotiation(Config, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts0 = proplists:get_value(server_opts, Config), + ServerOpts0 = proplists:get_value(server_rsa_opts, Config), ServerOpts = [{alpn_preferred_protocols, [<<"spdy/2">>]}, {next_protocols_advertised, [<<"spdy/3">>, <<"http/1.1">>]} | ServerOpts0], @@ -1620,8 +1630,8 @@ start_erlang_server_and_openssl_client_for_alpn_npn_negotiation(Config, Data, Ca start_erlang_client_and_openssl_server_for_npn_negotiation(Config, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), - ClientOpts0 = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_rsa_opts, Config), + ClientOpts0 = ssl_test_lib:ssl_options(client_rsa_opts, Config), ClientOpts = [{client_preferred_next_protocols, {client, [<<"spdy/2">>], <<"http/1.1">>}} | ClientOpts0], {ClientNode, _, Hostname} = ssl_test_lib:run_where(Config), @@ -1658,10 +1668,10 @@ start_erlang_client_and_openssl_server_for_npn_negotiation(Config, Data, Callbac start_erlang_server_and_openssl_client_for_npn_negotiation(Config, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts0 = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), ServerOpts = [{next_protocols_advertised, [<<"spdy/2">>]}, ServerOpts0], - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -1672,7 +1682,8 @@ start_erlang_server_and_openssl_client_for_npn_negotiation(Config, Data, Callbac Version = ssl_test_lib:protocol_version(Config), Exe = "openssl", - Args = ["s_client", "-nextprotoneg", "http/1.0,spdy/2", "-msg", "-connect", "localhost:" + Args = ["s_client", "-nextprotoneg", "http/1.0,spdy/2", "-msg", "-connect", + hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), ssl_test_lib:version_flag(Version)], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), @@ -1687,10 +1698,10 @@ start_erlang_server_and_openssl_client_for_npn_negotiation(Config, Data, Callbac start_erlang_server_and_openssl_client_with_opts(Config, ErlangServerOpts, OpenSSLClientOpts, Data, Callback) -> process_flag(trap_exit, true), - ServerOpts0 = ssl_test_lib:ssl_options(server_opts, Config), + ServerOpts0 = ssl_test_lib:ssl_options(server_rsa_opts, Config), ServerOpts = ErlangServerOpts ++ ServerOpts0, - {_, ServerNode, _} = ssl_test_lib:run_where(Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, @@ -1701,8 +1712,9 @@ start_erlang_server_and_openssl_client_with_opts(Config, ErlangServerOpts, OpenS Version = ssl_test_lib:protocol_version(Config), Exe = "openssl", - Args = ["s_client"] ++ OpenSSLClientOpts ++ ["-msg", "-connect", "localhost:" ++ integer_to_list(Port), - ssl_test_lib:version_flag(Version)], + Args = ["s_client"] ++ OpenSSLClientOpts ++ ["-msg", "-connect", + hostname_format(Hostname) ++ ":" ++ integer_to_list(Port), + ssl_test_lib:version_flag(Version)], OpenSslPort = ssl_test_lib:portable_open_port(Exe, Args), @@ -1854,3 +1866,11 @@ consume_port_exit(OpenSSLPort) -> {'EXIT', OpenSSLPort, _} -> ok end. + +hostname_format(Hostname) -> + case lists:member($., Hostname) of + true -> + Hostname; + false -> + "localhost" + end. diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index f6f3d18d6a..95af2b77a5 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -408,9 +408,9 @@ calls cannot be in the guard or body of the fun. Calls to built-in match specification functions is of course allowed:</p> <pre> -4> <input>ets:fun2ms(fun({M,N}) when N > X, is_atomm(M) -> M end).</input> +4> <input>ets:fun2ms(fun({M,N}) when N > X, my_fun(M) -> M end).</input> Error: fun containing local Erlang function calls -('is_atomm' called in guard) cannot be translated into match_spec +('my_fun' called in guard) cannot be translated into match_spec {error,transform_error} 5> <input>ets:fun2ms(fun({M,N}) when N > X, is_atom(M) -> M end).</input> [{{'$1','$2'},[{'>','$2',{const,3}},{is_atom,'$1'}],['$1']}]</pre> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index a7caa71dcb..8de6ed754f 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -1851,7 +1851,7 @@ handle_event(_, _, State, Data) -> </p> <note> <p> - Note that if the <c>gen_statem</c> is started trough + Note that if the <c>gen_statem</c> is started through <seealso marker="proc_lib"><c>proc_lib</c></seealso> and <seealso marker="#enter_loop/4"><c>enter_loop/4-6</c></seealso>, diff --git a/lib/stdlib/doc/src/lists.xml b/lib/stdlib/doc/src/lists.xml index 60dbae70c2..7efafedc82 100644 --- a/lib/stdlib/doc/src/lists.xml +++ b/lib/stdlib/doc/src/lists.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2016</year> + <year>1996</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -187,7 +187,7 @@ <desc> <p>Calls <c><anno>Fun</anno>(<anno>Elem</anno>)</c> on successive elements <c>Elem</c> of <c><anno>List1</anno></c>. - <c><anno>Fun</anno>/2</c> must return either a Boolean or a tuple + <c><anno>Fun</anno>/1</c> must return either a Boolean or a tuple <c>{true, <anno>Value</anno>}</c>. The function returns the list of elements for which <c><anno>Fun</anno></c> returns a new value, where a value of <c>true</c> is synonymous with diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 46454e9b80..604d758db3 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,38 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> A bug in <c>proc_lib:format()</c> introduced in + Erlang/OTP 20.0 is corrected. </p> + <p> + Own Id: OTP-14482 Aux Id: PR-1488 </p> + </item> + <item> + <p> + Fix string:len/1 to be compatible with previous versions.</p> + <p> + Own Id: OTP-14487 Aux Id: ERIERL-40 </p> + </item> + <item> + <p> + In OTP-20.0, the behavior of c, make, and ct_make was + changed so that in some cases the beam files by default + would be written to the directory where the source files + were found. This is now changed back to the old behavior + so beam files are by default written to current + directory.</p> + <p> + Own Id: OTP-14489 Aux Id: ERL-438 </p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.4</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -400,7 +432,7 @@ marker="erts:erl"><c>erl</c></seealso> command.</p> <p> See <url - href="http://pcre.org/original/changelog.txt"><c>http://pcre.org/original/changelog.txt</c></url> + href="http://pcre.org/original/changelog.txt">http://pcre.org/original/changelog.txt</url> for information about changes made to PCRE between the versions 8.33 and 8.40.</p> <p> diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index e06d7e467d..a68fb7d55f 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -66,7 +66,7 @@ <p>Jump function: equivalent to 2^64 calls</p> <p> This is a corrected version of the previous default algorithm, - that now has been superseeded by Xoroshiro116+ (<c>exrop</c>). + that now has been superseded by Xoroshiro116+ (<c>exrop</c>). Since there is no native 58 bit rotate instruction this algorithm executes a little (say < 15%) faster than <c>exrop</c>. See the diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 079b761463..a237eaa489 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -1603,7 +1603,7 @@ foldl_2(I, E, A, Ix, F, D, N, R, S) -> Ix + S, F, D, N, R, S). -spec foldl_3(pos_integer(), _, A, array_indx(), - fun((array_indx, _, A) -> B), integer()) -> B. + fun((array_indx(), _, A) -> B), integer()) -> B. foldl_3(I, E, A, Ix, F, N) when I =< N -> foldl_3(I+1, E, F(Ix, element(I, E), A), Ix+1, F, N); diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 4ab9234b81..c04a201ce1 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -255,7 +255,7 @@ safe_recompile(File, Options, BeamFile) -> compile_and_load(File, Opts0) when is_list(Opts0) -> Opts = [report_errors, report_warnings | ensure_from(filename:extension(File), - ensure_outdir(filename:dirname(File), Opts0))], + ensure_outdir(".", Opts0))], case compile:file(File, Opts) of {ok,Mod} -> %Listing file. purge_and_load(Mod, File, Opts); diff --git a/lib/stdlib/src/edlin.erl b/lib/stdlib/src/edlin.erl index 71e8471c45..64d5a71f3c 100644 --- a/lib/stdlib/src/edlin.erl +++ b/lib/stdlib/src/edlin.erl @@ -83,7 +83,7 @@ edit_line(Cs, {line,P,L,M}) -> edit_line1(Cs, {line,P,L,{blink,N}}) -> edit(Cs, P, L, none, [{move_rel,N}]); edit_line1(Cs, {line,P,{[],[]},none}) -> - {more_chars, {line,P,{lists:reverse(Cs),[]},none},[{put_chars, unicode, Cs}]}; + {more_chars, {line,P,{string:reverse(Cs),[]},none},[{put_chars, unicode, Cs}]}; edit_line1(Cs, {line,P,L,M}) -> edit(Cs, P, L, M, []). @@ -93,14 +93,14 @@ edit([C|Cs], P, {Bef,Aft}, Prefix, Rs0) -> case key_map(C, Prefix) of meta -> edit(Cs, P, {Bef,Aft}, meta, Rs0); - meta_o -> - edit(Cs, P, {Bef,Aft}, meta_o, Rs0); - meta_csi -> - edit(Cs, P, {Bef,Aft}, meta_csi, Rs0); - meta_meta -> - edit(Cs, P, {Bef,Aft}, meta_meta, Rs0); - {csi, _} = Csi -> - edit(Cs, P, {Bef,Aft}, Csi, Rs0); + meta_o -> + edit(Cs, P, {Bef,Aft}, meta_o, Rs0); + meta_csi -> + edit(Cs, P, {Bef,Aft}, meta_csi, Rs0); + meta_meta -> + edit(Cs, P, {Bef,Aft}, meta_meta, Rs0); + {csi, _} = Csi -> + edit(Cs, P, {Bef,Aft}, Csi, Rs0); meta_left_sq_bracket -> edit(Cs, P, {Bef,Aft}, meta_left_sq_bracket, Rs0); search_meta -> @@ -110,8 +110,8 @@ edit([C|Cs], P, {Bef,Aft}, Prefix, Rs0) -> ctlx -> edit(Cs, P, {Bef,Aft}, ctlx, Rs0); new_line -> - {done, reverse(Bef, Aft ++ "\n"), Cs, - reverse(Rs0, [{move_rel,length(Aft)},{put_chars,unicode,"\n"}])}; + {done, get_line(Bef, Aft ++ "\n"), Cs, + reverse(Rs0, [{move_rel,cp_len(Aft)},{put_chars,unicode,"\n"}])}; redraw_line -> Rs1 = erase(P, Bef, Aft, Rs0), Rs = redraw(P, Bef, Aft, Rs1), @@ -157,7 +157,7 @@ edit([], P, L, {blink,N}, Rs) -> edit([], P, L, Prefix, Rs) -> {more_chars,{line,P,L,Prefix},reverse(Rs)}; edit(eof, _, {Bef,Aft}, _, Rs) -> - {done,reverse(Bef, Aft),[],reverse(Rs, [{move_rel,length(Aft)}])}. + {done,get_line(Bef, Aft),[],reverse(Rs, [{move_rel,cp_len(Aft)}])}. %% %% Assumes that arg is a string %% %% Horizontal whitespace only. @@ -279,11 +279,21 @@ key_map(C, search) -> {insert_search,C}; key_map(C, _) -> {undefined,C}. %% do_op(Action, Before, After, Requests) - -do_op({insert,C}, Bef, [], Rs) -> - {{[C|Bef],[]},[{put_chars, unicode,[C]}|Rs]}; -do_op({insert,C}, Bef, Aft, Rs) -> - {{[C|Bef],Aft},[{insert_chars, unicode, [C]}|Rs]}; +%% Before and After are of lists of type string:grapheme_cluster() +do_op({insert,C}, [], [], Rs) -> + {{[C],[]},[{put_chars, unicode,[C]}|Rs]}; +do_op({insert,C}, [Bef|Bef0], [], Rs) -> + case string:to_graphemes([Bef,C]) of + [GC] -> {{[GC|Bef0],[]},[{put_chars, unicode,[C]}|Rs]}; + _ -> {{[C,Bef|Bef0],[]},[{put_chars, unicode,[C]}|Rs]} + end; +do_op({insert,C}, [], Aft, Rs) -> + {{[C],Aft},[{insert_chars, unicode,[C]}|Rs]}; +do_op({insert,C}, [Bef|Bef0], Aft, Rs) -> + case string:to_graphemes([Bef,C]) of + [GC] -> {{[GC|Bef0],Aft},[{insert_chars, unicode,[C]}|Rs]}; + _ -> {{[C,Bef|Bef0],Aft},[{insert_chars, unicode,[C]}|Rs]} + end; %% Search mode prompt always looks like (search)`$TERMS': $RESULT. %% the {insert_search, _} handlings allow to share this implementation %% correctly with group.erl. This module provides $TERMS, and group.erl @@ -299,13 +309,13 @@ do_op({insert_search, C}, Bef, [], Rs) -> [{insert_chars, unicode, [C]++Aft}, {delete_chars,-3} | Rs], search}; do_op({insert_search, C}, Bef, Aft, Rs) -> - Offset= length(Aft), + Offset= cp_len(Aft), NAft = "': ", {{[C|Bef],NAft}, [{insert_chars, unicode, [C]++NAft}, {delete_chars,-Offset} | Rs], search}; do_op({search, backward_delete_char}, [_|Bef], Aft, Rs) -> - Offset= length(Aft)+1, + Offset= cp_len(Aft)+1, NAft = "': ", {{Bef,NAft}, [{insert_chars, unicode, NAft}, {delete_chars,-Offset}|Rs], @@ -314,13 +324,13 @@ do_op({search, backward_delete_char}, [], _Aft, Rs) -> Aft="': ", {{[],Aft}, Rs, search}; do_op({search, skip_up}, Bef, Aft, Rs) -> - Offset= length(Aft), + Offset= cp_len(Aft), NAft = "': ", {{[$\^R|Bef],NAft}, % we insert ^R as a flag to whoever called us [{insert_chars, unicode, NAft}, {delete_chars,-Offset}|Rs], search}; do_op({search, skip_down}, Bef, Aft, Rs) -> - Offset= length(Aft), + Offset= cp_len(Aft), NAft = "': ", {{[$\^S|Bef],NAft}, % we insert ^S as a flag to whoever called us [{insert_chars, unicode, NAft}, {delete_chars,-Offset}|Rs], @@ -328,12 +338,12 @@ do_op({search, skip_down}, Bef, Aft, Rs) -> do_op({search, search_found}, _Bef, Aft, Rs) -> "': "++NAft = Aft, {{[],NAft}, - [{put_chars, unicode, "\n"}, {move_rel,-length(Aft)} | Rs], + [{put_chars, unicode, "\n"}, {move_rel,-cp_len(Aft)} | Rs], search_found}; do_op({search, search_quit}, _Bef, Aft, Rs) -> "': "++NAft = Aft, {{[],NAft}, - [{put_chars, unicode, "\n"}, {move_rel,-length(Aft)} | Rs], + [{put_chars, unicode, "\n"}, {move_rel,-cp_len(Aft)} | Rs], search_quit}; %% do blink after $$ do_op({blink,C,M}, Bef=[$$,$$|_], Aft, Rs) -> @@ -361,14 +371,16 @@ do_op(auto_blink, Bef, Aft, Rs) -> N -> {blink,N+1,{Bef,Aft}, [{move_rel,-(N+1)}|Rs]} end; -do_op(forward_delete_char, Bef, [_|Aft], Rs) -> - {{Bef,Aft},[{delete_chars,1}|Rs]}; -do_op(backward_delete_char, [_|Bef], Aft, Rs) -> - {{Bef,Aft},[{delete_chars,-1}|Rs]}; +do_op(forward_delete_char, Bef, [GC|Aft], Rs) -> + {{Bef,Aft},[{delete_chars,gc_len(GC)}|Rs]}; +do_op(backward_delete_char, [GC|Bef], Aft, Rs) -> + {{Bef,Aft},[{delete_chars,-gc_len(GC)}|Rs]}; do_op(transpose_char, [C1,C2|Bef], [], Rs) -> - {{[C2,C1|Bef],[]},[{put_chars, unicode,[C1,C2]},{move_rel,-2}|Rs]}; + Len = gc_len(C1)+gc_len(C2), + {{[C2,C1|Bef],[]},[{put_chars, unicode,[C1,C2]},{move_rel,-Len}|Rs]}; do_op(transpose_char, [C2|Bef], [C1|Aft], Rs) -> - {{[C2,C1|Bef],Aft},[{put_chars, unicode,[C1,C2]},{move_rel,-1}|Rs]}; + Len = gc_len(C2), + {{[C2,C1|Bef],Aft},[{put_chars, unicode,[C1,C2]},{move_rel,-Len}|Rs]}; do_op(kill_word, Bef, Aft0, Rs) -> {Aft1,Kill0,N0} = over_non_word(Aft0, [], 0), {Aft,Kill,N} = over_word(Aft1, Kill0, N0), @@ -381,7 +393,7 @@ do_op(backward_kill_word, Bef0, Aft, Rs) -> {{Bef,Aft},[{delete_chars,-N}|Rs]}; do_op(kill_line, Bef, Aft, Rs) -> put(kill_buffer, Aft), - {{Bef,[]},[{delete_chars,length(Aft)}|Rs]}; + {{Bef,[]},[{delete_chars,cp_len(Aft)}|Rs]}; do_op(yank, Bef, [], Rs) -> Kill = get(kill_buffer), {{reverse(Kill, Bef),[]},[{put_chars, unicode,Kill}|Rs]}; @@ -389,9 +401,9 @@ do_op(yank, Bef, Aft, Rs) -> Kill = get(kill_buffer), {{reverse(Kill, Bef),Aft},[{insert_chars, unicode,Kill}|Rs]}; do_op(forward_char, Bef, [C|Aft], Rs) -> - {{[C|Bef],Aft},[{move_rel,1}|Rs]}; + {{[C|Bef],Aft},[{move_rel,gc_len(C)}|Rs]}; do_op(backward_char, [C|Bef], Aft, Rs) -> - {{Bef,[C|Aft]},[{move_rel,-1}|Rs]}; + {{Bef,[C|Aft]},[{move_rel,-gc_len(C)}|Rs]}; do_op(forward_word, Bef0, Aft0, Rs) -> {Aft1,Bef1,N0} = over_non_word(Aft0, Bef0, 0), {Aft,Bef,N} = over_word(Aft1, Bef1, N0), @@ -401,16 +413,16 @@ do_op(backward_word, Bef0, Aft0, Rs) -> {Bef,Aft,N} = over_word(Bef1, Aft1, N0), {{Bef,Aft},[{move_rel,-N}|Rs]}; do_op(beginning_of_line, [C|Bef], Aft, Rs) -> - {{[],reverse(Bef, [C|Aft])},[{move_rel,-(length(Bef)+1)}|Rs]}; + {{[],reverse(Bef, [C|Aft])},[{move_rel,-(cp_len(Bef)+1)}|Rs]}; do_op(beginning_of_line, [], Aft, Rs) -> {{[],Aft},Rs}; do_op(end_of_line, Bef, [C|Aft], Rs) -> - {{reverse(Aft, [C|Bef]),[]},[{move_rel,length(Aft)+1}|Rs]}; + {{reverse(Aft, [C|Bef]),[]},[{move_rel,cp_len(Aft)+1}|Rs]}; do_op(end_of_line, Bef, [], Rs) -> {{Bef,[]},Rs}; do_op(ctlu, Bef, Aft, Rs) -> put(kill_buffer, reverse(Bef)), - {{[], Aft}, [{delete_chars, -length(Bef)} | Rs]}; + {{[], Aft}, [{delete_chars, -cp_len(Bef)} | Rs]}; do_op(beep, Bef, Aft, Rs) -> {{Bef,Aft},[beep|Rs]}; do_op(_, Bef, Aft, Rs) -> @@ -436,7 +448,7 @@ over_word(Cs, Stack, N) -> until_quote([$\'|Cs], Stack, N) -> {Cs, [$\'|Stack], N+1}; until_quote([C|Cs], Stack, N) -> - until_quote(Cs, [C|Stack], N+1). + until_quote(Cs, [C|Stack], N+gc_len(C)). over_word1([$\'=C|Cs], Stack, N) -> until_quote(Cs, [C|Stack], N+1); @@ -445,7 +457,7 @@ over_word1(Cs, Stack, N) -> over_word2([C|Cs], Stack, N) -> case word_char(C) of - true -> over_word2(Cs, [C|Stack], N+1); + true -> over_word2(Cs, [C|Stack], N+gc_len(C)); false -> {[C|Cs],Stack,N} end; over_word2([], Stack, N) when is_integer(N) -> @@ -454,7 +466,7 @@ over_word2([], Stack, N) when is_integer(N) -> over_non_word([C|Cs], Stack, N) -> case word_char(C) of true -> {[C|Cs],Stack,N}; - false -> over_non_word(Cs, [C|Stack], N+1) + false -> over_non_word(Cs, [C|Stack], N+gc_len(C)) end; over_non_word([], Stack, N) -> {[],Stack,N}. @@ -465,6 +477,7 @@ word_char(C) when C >= $a, C =< $z -> true; word_char(C) when C >= $ß, C =< $ÿ, C =/= $÷ -> true; word_char(C) when C >= $0, C =< $9 -> true; word_char(C) when C =:= $_ -> true; +word_char([_|_]) -> true; %% Is grapheme word_char(_) -> false. %% over_white(Chars, InitialStack, InitialCount) -> @@ -488,8 +501,8 @@ over_paren(Chars, Paren, Match) -> over_paren([C,$$,$$|Cs], Paren, Match, D, N, L) -> over_paren([C|Cs], Paren, Match, D, N+2, L); -over_paren([_,$$|Cs], Paren, Match, D, N, L) -> - over_paren(Cs, Paren, Match, D, N+2, L); +over_paren([GC,$$|Cs], Paren, Match, D, N, L) -> + over_paren(Cs, Paren, Match, D, N+1+gc_len(GC), L); over_paren([Match|_], _Paren, Match, 1, N, _) -> N; over_paren([Match|Cs], Paren, Match, D, N, [Match|L]) -> @@ -518,8 +531,8 @@ over_paren([$[|_], _, _, _, _, _) -> over_paren([${|_], _, _, _, _, _) -> beep; -over_paren([_|Cs], Paren, Match, D, N, L) -> - over_paren(Cs, Paren, Match, D, N+1, L); +over_paren([GC|Cs], Paren, Match, D, N, L) -> + over_paren(Cs, Paren, Match, D, N+gc_len(GC), L); over_paren([], _, _, _, _, _) -> 0. @@ -529,8 +542,8 @@ over_paren_auto(Chars) -> over_paren_auto([C,$$,$$|Cs], D, N, L) -> over_paren_auto([C|Cs], D, N+2, L); -over_paren_auto([_,$$|Cs], D, N, L) -> - over_paren_auto(Cs, D, N+2, L); +over_paren_auto([GC,$$|Cs], D, N, L) -> + over_paren_auto(Cs, D, N+1+gc_len(GC), L); over_paren_auto([$(|_], _, N, []) -> {N, $)}; @@ -553,8 +566,8 @@ over_paren_auto([$[|Cs], D, N, [$[|L]) -> over_paren_auto([${|Cs], D, N, [${|L]) -> over_paren_auto(Cs, D, N+1, L); -over_paren_auto([_|Cs], D, N, L) -> - over_paren_auto(Cs, D, N+1, L); +over_paren_auto([GC|Cs], D, N, L) -> + over_paren_auto(Cs, D, N+gc_len(GC), L); over_paren_auto([], _, _, _) -> 0. @@ -574,28 +587,43 @@ erase_inp({line,_,{Bef,Aft},_}) -> reverse(erase([], Bef, Aft, [])). erase(Pbs, Bef, Aft, Rs) -> - [{delete_chars,-length(Pbs)-length(Bef)},{delete_chars,length(Aft)}|Rs]. + [{delete_chars,-cp_len(Pbs)-cp_len(Bef)},{delete_chars,cp_len(Aft)}|Rs]. redraw_line({line,Pbs,{Bef,Aft},_}) -> reverse(redraw(Pbs, Bef, Aft, [])). redraw(Pbs, Bef, Aft, Rs) -> - [{move_rel,-length(Aft)},{put_chars, unicode,reverse(Bef, Aft)},{put_chars, unicode,Pbs}|Rs]. + [{move_rel,-cp_len(Aft)},{put_chars, unicode,reverse(Bef, Aft)},{put_chars, unicode,Pbs}|Rs]. length_before({line,Pbs,{Bef,_Aft},_}) -> - length(Pbs) + length(Bef). + cp_len(Pbs) + cp_len(Bef). length_after({line,_,{_Bef,Aft},_}) -> - length(Aft). + cp_len(Aft). prompt({line,Pbs,_,_}) -> Pbs. current_line({line,_,{Bef, Aft},_}) -> - reverse(Bef, Aft ++ "\n"). + get_line(Bef, Aft ++ "\n"). current_chars({line,_,{Bef,Aft},_}) -> - reverse(Bef, Aft). + get_line(Bef, Aft). + +get_line(Bef, Aft) -> + unicode:characters_to_list(reverse(Bef, Aft)). + +%% Grapheme length in codepoints +gc_len(CP) when is_integer(CP) -> 1; +gc_len(CPs) when is_list(CPs) -> length(CPs). + +%% String length in codepoints +cp_len(Str) -> + cp_len(Str, 0). + +cp_len([GC|R], Len) -> + cp_len(R, Len + gc_len(GC)); +cp_len([], Len) -> Len. %% %% expand(CurrentBefore) -> %% %% {yes,Expansion} | no diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index fcfd0d8493..65ba343368 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -194,8 +194,6 @@ format_error({bad_nowarn_bif_clash,{F,A}}) -> format_error(disallowed_nowarn_bif_clash) -> io_lib:format("compile directive nowarn_bif_clash is no longer allowed,~n" " - use explicit module names or -compile({no_auto_import, [F/A]})", []); -format_error({bad_nowarn_deprecated_function,{M,F,A}}) -> - io_lib:format("~tw:~tw/~w is not a deprecated function", [M,F,A]); format_error({bad_on_load,Term}) -> io_lib:format("badly formed on_load attribute: ~tw", [Term]); format_error(multiple_on_loads) -> @@ -856,14 +854,11 @@ not_deprecated(Forms, St0) -> {nowarn_deprecated_function, MFAs0} <- lists:flatten([Args]), MFA <- lists:flatten([MFAs0])], Nowarn = [MFA || {MFA,_L} <- MFAsL], - Bad = [MFAL || {{M,F,A},_L}=MFAL <- MFAsL, - otp_internal:obsolete(M, F, A) =:= no], - St1 = func_line_warning(bad_nowarn_deprecated_function, Bad, St0), ML = [{M,L} || {{M,_F,_A},L} <- MFAsL, is_atom(M)], - St3 = foldl(fun ({M,L}, St2) -> + St1 = foldl(fun ({M,L}, St2) -> check_module_name(M, L, St2) - end, St1, ML), - St3#lint{not_deprecated = ordsets:from_list(Nowarn)}. + end, St0, ML), + St1#lint{not_deprecated = ordsets:from_list(Nowarn)}. %% The nowarn_bif_clash directive is not only deprecated, it's actually an error from R14A disallowed_compile_flags(Forms, St0) -> diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index ee5e7a11bf..b0064aadb8 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -598,8 +598,6 @@ lexpr({'fun',_,{clauses,Cs},Extra}, _Prec, Opts) -> lexpr({named_fun,_,Name,Cs,Extra}, _Prec, Opts) -> {force_nl,fun_info(Extra), {list,[{first,['fun', " "],fun_clauses(Cs, Opts, {named, Name})},'end']}}; -lexpr({'query',_,Lc}, _Prec, Opts) -> - {list,[{step,leaf("query"),lexpr(Lc, 0, Opts)},'end']}; lexpr({call,_,{remote,_,{atom,_,M},{atom,_,F}=N}=Name,Args}, Prec, Opts) -> case erl_internal:bif(M, F, length(Args)) of true -> diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 9e9c0dc413..c59db903dc 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -63,9 +63,9 @@ obsolete_1(gen_fsm, start, 4) -> {deprecated, {gen_statem, start, 4}}; obsolete_1(gen_fsm, start_link, 3) -> - {deprecated, {gen_statem, start, 3}}; + {deprecated, {gen_statem, start_link, 3}}; obsolete_1(gen_fsm, start_link, 4) -> - {deprecated, {gen_statem, start, 4}}; + {deprecated, {gen_statem, start_link, 4}}; obsolete_1(gen_fsm, stop, 1) -> {deprecated, {gen_statem, stop, 1}}; @@ -83,9 +83,9 @@ obsolete_1(gen_fsm, reply, 2) -> {deprecated, {gen_statem, reply, 2}}; obsolete_1(gen_fsm, send_event, 2) -> - {deprecated, {gen_statem, cast, 1}}; + {deprecated, {gen_statem, cast, 2}}; obsolete_1(gen_fsm, send_all_state_event, 2) -> - {deprecated, {gen_statem, cast, 1}}; + {deprecated, {gen_statem, cast, 2}}; obsolete_1(gen_fsm, sync_send_event, 2) -> {deprecated, {gen_statem, call, 2}}; @@ -98,11 +98,11 @@ obsolete_1(gen_fsm, sync_send_all_state_event, 3) -> {deprecated, {gen_statem, call, 3}}; obsolete_1(gen_fsm, start_timer, 2) -> - {deprecated, {erlang, start_timer, 2}}; + {deprecated, {erlang, start_timer, 3}}; obsolete_1(gen_fsm, cancel_timer, 1) -> {deprecated, {erlang, cancel_timer, 1}}; obsolete_1(gen_fsm, send_event_after, 2) -> - {deprecated, {erlang, send_after, 2}}; + {deprecated, {erlang, send_after, 3}}; %% *** CRYPTO added in OTP 20 *** @@ -112,7 +112,7 @@ obsolete_1(crypto, rand_uniform, 2) -> %% *** CRYPTO added in OTP 19 *** obsolete_1(crypto, rand_bytes, 1) -> - {deprecated, {crypto, strong_rand_bytes, 1}}; + {removed, {crypto, strong_rand_bytes, 1}, "20.0"}; %% *** CRYPTO added in R16B01 *** @@ -485,10 +485,6 @@ obsolete_1(wxPaintDC, new, 0) -> {deprecated,"deprecated function not available in wxWidgets-2.9 and later"}; obsolete_1(wxWindowDC, new, 0) -> {deprecated,"deprecated function not available in wxWidgets-2.9 and later"}; -obsolete_1(wxGraphicsContext, createLinearGradientBrush, 7) -> - {deprecated,"deprecated function not available in wxWidgets-2.9 and later"}; -obsolete_1(wxGraphicsContext, createRadialGradientBrush, 8) -> - {deprecated,"deprecated function not available in wxWidgets-2.9 and later"}; obsolete_1(wxGraphicsRenderer, createLinearGradientBrush, 7) -> {deprecated,"deprecated function not available in wxWidgets-2.9 and later"}; obsolete_1(wxGraphicsRenderer, createRadialGradientBrush, 8) -> diff --git a/lib/stdlib/src/proc_lib.erl b/lib/stdlib/src/proc_lib.erl index 9ce8e7d60e..d4d1bdccec 100644 --- a/lib/stdlib/src/proc_lib.erl +++ b/lib/stdlib/src/proc_lib.erl @@ -779,11 +779,13 @@ format_link_report([Link|Reps], Indent, Extra) -> LinkIndent = [" ",Indent], [Indent,"neighbour:\n",format_report(Rep, LinkIndent, Extra)| format_link_report(Reps, Indent, Extra)]; -format_link_report([], _, _) -> - []. +format_link_report(Rep, Indent, Extra) -> + format_report(Rep, Indent, Extra). format_report(Rep, Indent, Extra) when is_list(Rep) -> format_rep(Rep, Indent, Extra); +format_report(Rep, Indent, {Enc,unlimited}) -> + io_lib:format("~s~"++modifier(Enc)++"p~n", [Indent, Rep]); format_report(Rep, Indent, {Enc,Depth}) -> io_lib:format("~s~"++modifier(Enc)++"P~n", [Indent, Rep, Depth]). diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index 6eafc7b209..26b3960f4f 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -727,7 +727,7 @@ result_will_be_saved() -> used_record_defs(E, RT) -> %% Be careful to return a list where used records come before %% records that use them. The linter wants them ordered that way. - UR = case used_records(E, [], RT) of + UR = case used_records(E, [], RT, []) of [] -> []; L0 -> @@ -737,13 +737,19 @@ used_record_defs(E, RT) -> end, record_defs(RT, UR). -used_records(E, U0, RT) -> +used_records(E, U0, RT, Skip) -> case used_records(E) of {name,Name,E1} -> - U = used_records(ets:lookup(RT, Name), [Name | U0], RT), - used_records(E1, U, RT); + U = case lists:member(Name, Skip) of + true -> + U0; + false -> + R = ets:lookup(RT, Name), + used_records(R, [Name | U0], RT, [Name | Skip]) + end, + used_records(E1, U, RT, Skip); {expr,[E1 | Es]} -> - used_records(Es, used_records(E1, U0, RT), RT); + used_records(Es, used_records(E1, U0, RT, Skip), RT, Skip); _ -> U0 end. diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 3100504a80..e4e3fb83e9 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -18,7 +18,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.* + [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.* %% Down to - max one major revision back - [{<<"3\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.* + [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.* }. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index 6f7009b5d9..4972da297d 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -384,7 +384,7 @@ to_float(String) -> end. to_number(String, Number, Rest, List, _Tail) when is_binary(String) -> - BSz = length(List)-length(Rest), + BSz = erlang:length(List)-erlang:length(Rest), <<_:BSz/binary, Cont/binary>> = String, {Number, Cont}; to_number(_, Number, Rest, _, Tail) -> @@ -1344,7 +1344,7 @@ bin_search_str(Bin0, Start, Cont, [CP|_]=SearchCPs) -> String :: string(), Length :: non_neg_integer(). -len(S) -> length(S). +len(S) -> erlang:length(S). %% equal(String1, String2) %% Test if 2 strings are equal. @@ -1689,7 +1689,7 @@ left(String, Len) when is_integer(Len) -> left(String, Len, $\s). Character :: char(). left(String, Len, Char) when is_integer(Char) -> - Slen = length(String), + Slen = erlang:length(String), if Slen > Len -> substr(String, 1, Len); Slen < Len -> l_pad(String, Len-Slen, Char); @@ -1714,7 +1714,7 @@ right(String, Len) when is_integer(Len) -> right(String, Len, $\s). Character :: char(). right(String, Len, Char) when is_integer(Char) -> - Slen = length(String), + Slen = erlang:length(String), if Slen > Len -> substr(String, Slen-Len+1); Slen < Len -> r_pad(String, Len-Slen, Char); @@ -1741,7 +1741,7 @@ centre(String, Len) when is_integer(Len) -> centre(String, Len, $\s). centre(String, 0, Char) when is_list(String), is_integer(Char) -> []; % Strange cases to centre string centre(String, Len, Char) when is_integer(Char) -> - Slen = length(String), + Slen = erlang:length(String), if Slen > Len -> substr(String, (Slen-Len) div 2 + 1, Len); Slen < Len -> diff --git a/lib/stdlib/test/c_SUITE.erl b/lib/stdlib/test/c_SUITE.erl index 4bd32a30f8..f01988478c 100644 --- a/lib/stdlib/test/c_SUITE.erl +++ b/lib/stdlib/test/c_SUITE.erl @@ -21,7 +21,9 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). -export([c_1/1, c_2/1, c_3/1, c_4/1, nc_1/1, nc_2/1, nc_3/1, nc_4/1, - ls/1, memory/1]). + c_default_outdir_1/1, c_default_outdir_2/1, + nc_default_outdir_1/1, nc_default_outdir_2/1, + ls/1, memory/1]). -include_lib("common_test/include/ct.hrl"). @@ -30,7 +32,10 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [c_1, c_2, c_3, c_4, nc_1, nc_2, nc_3, nc_4, ls, memory]. + [c_1, c_2, c_3, c_4, nc_1, nc_2, nc_3, nc_4, + c_default_outdir_1, c_default_outdir_2, + nc_default_outdir_1, nc_default_outdir_2, + ls, memory]. groups() -> []. @@ -124,6 +129,50 @@ nc_4(Config) when is_list(Config) -> Result = nc(R,[{outdir,W}]), {ok, m} = Result. +c_default_outdir_1(Config) -> + R = filename:join(proplists:get_value(data_dir, Config), "m.erl"), + W = proplists:get_value(priv_dir, Config), + file:set_cwd(W), + Obj = "m" ++ code:objfile_extension(), + _ = file:delete(Obj), + false = filelib:is_file(Obj), + Result = c:c(R), + {ok, m} = Result, + true = filelib:is_file(Obj). + +c_default_outdir_2(Config) -> + R = filename:join(proplists:get_value(data_dir, Config), "m"), + W = proplists:get_value(priv_dir, Config), + file:set_cwd(W), + Obj = "m" ++ code:objfile_extension(), + _ = file:delete(Obj), + false = filelib:is_file(Obj), + Result = c:c(R), + {ok, m} = Result, + true = filelib:is_file(Obj). + +nc_default_outdir_1(Config) -> + R = filename:join(proplists:get_value(data_dir, Config), "m.erl"), + W = proplists:get_value(priv_dir, Config), + file:set_cwd(W), + Obj = "m" ++ code:objfile_extension(), + _ = file:delete(Obj), + false = filelib:is_file(Obj), + Result = c:nc(R), + {ok, m} = Result, + true = filelib:is_file(Obj). + +nc_default_outdir_2(Config) -> + R = filename:join(proplists:get_value(data_dir, Config), "m"), + W = proplists:get_value(priv_dir, Config), + file:set_cwd(W), + Obj = "m" ++ code:objfile_extension(), + _ = file:delete(Obj), + false = filelib:is_file(Obj), + Result = c:nc(R), + {ok, m} = Result, + true = filelib:is_file(Obj). + ls(Config) when is_list(Config) -> Directory = proplists:get_value(data_dir, Config), ok = c:ls(Directory), diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 6a75eaa737..cb1cceb8db 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -66,7 +66,7 @@ otp_11851/1,otp_11879/1,otp_13230/1, record_errors/1, otp_11879_cont/1, non_latin1_module/1, otp_14323/1, - get_stacktrace/1, otp_14285/1]). + get_stacktrace/1, otp_14285/1, otp_14378/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -87,7 +87,7 @@ all() -> maps, maps_type, maps_parallel_match, otp_11851, otp_11879, otp_13230, record_errors, otp_11879_cont, non_latin1_module, otp_14323, - get_stacktrace, otp_14285]. + get_stacktrace, otp_14285, otp_14378]. groups() -> [{unused_vars_warn, [], @@ -2054,12 +2054,10 @@ otp_5362(Config) when is_list(Config) -> spawn(A). ">>, {[nowarn_unused_function]}, - {error,[{3,erl_lint,disallowed_nowarn_bif_clash}, - {4,erl_lint,disallowed_nowarn_bif_clash}, - {4,erl_lint,{bad_nowarn_bif_clash,{spawn,2}}}], - [{5,erl_lint,{bad_nowarn_deprecated_function,{3,now,-1}}}, - {5,erl_lint,{bad_nowarn_deprecated_function,{erlang,now,-1}}}, - {5,erl_lint,{bad_nowarn_deprecated_function,{{a,b,c},now,-1}}}]} + {errors,[{3,erl_lint,disallowed_nowarn_bif_clash}, + {4,erl_lint,disallowed_nowarn_bif_clash}, + {4,erl_lint,{bad_nowarn_bif_clash,{spawn,2}}}], + []} }, {otp_5362_8, @@ -3937,10 +3935,6 @@ non_latin1_module(Config) -> UndefBehav = {undefined_behaviour,'кирилли́ческий атом'}, "behaviour 'кирилли́ческий атом' undefined" = format_error(UndefBehav), - BadDepr = {bad_nowarn_deprecated_function, - {'кирилли́ческий атом','кирилли́ческий атом',18}}, - "'кирилли́ческий атом':'кирилли́ческий атом'/18 is not a deprecated " - "function" = format_error(BadDepr), Ts = [{non_latin1_module, <<" %% Report uses of module names with non-Latin-1 characters. @@ -3951,9 +3945,6 @@ non_latin1_module(Config) -> -callback 'кирилли́ческий атом':'кирилли́ческий атом'() -> a. - -compile([{nowarn_deprecated_function, - [{'кирилли́ческий атом','кирилли́ческий атом',18}]}]). - %% erl_lint:gexpr/3 is not extended to check module name here: t1() when 'кирилли́ческий атом':'кирилли́ческий атом'(1) -> b. @@ -3977,16 +3968,14 @@ non_latin1_module(Config) -> {6,erl_lint,non_latin1_module_unsupported}, {8,erl_lint,non_latin1_module_unsupported}, {8,erl_lint,BadCallback}, - {10,erl_lint,non_latin1_module_unsupported}, - {14,erl_lint,illegal_guard_expr}, - {18,erl_lint,non_latin1_module_unsupported}, + {11,erl_lint,illegal_guard_expr}, + {15,erl_lint,non_latin1_module_unsupported}, + {17,erl_lint,non_latin1_module_unsupported}, {20,erl_lint,non_latin1_module_unsupported}, {23,erl_lint,non_latin1_module_unsupported}, - {26,erl_lint,non_latin1_module_unsupported}, - {28,erl_lint,non_latin1_module_unsupported}], + {25,erl_lint,non_latin1_module_unsupported}], [{5,erl_lint,UndefBehav}, - {6,erl_lint,UndefBehav}, - {10,erl_lint,BadDepr}]}}], + {6,erl_lint,UndefBehav}]}}], run(Config, Ts), ok. @@ -4000,6 +3989,22 @@ do_non_latin1_module(Mod) -> ok. +otp_14378(Config) -> + Ts = [ + {otp_14378_1, + <<"-export([t/0]). + -compile({nowarn_deprecated_function,{erlang,now,1}}). + t() -> + erlang:now().">>, + [], + {warnings,[{4,erl_lint, + {deprecated,{erlang,now,0}, + "Deprecated BIF. See the \"Time and Time Correction" + " in Erlang\" chapter of the ERTS User's Guide" + " for more information."}}]}}], + [] = run(Config, Ts), + ok. + %% OTP-14323: Check the dialyzer attribute. otp_14323(Config) -> Ts = [ diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 05451a83fb..5a5e282998 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -2283,13 +2283,8 @@ write_concurrency(Config) when is_list(Config) -> NoHashMem = ets:info(No7,memory), NoHashMem = ets:info(No8,memory), - case erlang:system_info(smp_support) of - true -> - true = YesMem > NoHashMem, - true = YesMem > NoTreeMem; - false -> - true = YesMem =:= NoHashMem - end, + true = YesMem > NoHashMem, + true = YesMem > NoTreeMem, {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency,foo}])), {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency}])), @@ -5912,16 +5907,11 @@ add_lists([E1|T1], [E2|T2], Acc) -> run_smp_workers(InitF,ExecF,FiniF,Laps) -> run_smp_workers(InitF,ExecF,FiniF,Laps, 0). run_smp_workers(InitF,ExecF,FiniF,Laps, Exclude) -> - case erlang:system_info(smp_support) of - true -> - case erlang:system_info(schedulers_online) of - N when N > Exclude -> - run_workers_do(InitF,ExecF,FiniF,Laps, N - Exclude); - _ -> - {skipped, "Too few schedulers online"} - end; - false -> - {skipped,"No smp support"} + case erlang:system_info(schedulers_online) of + N when N > Exclude -> + run_workers_do(InitF,ExecF,FiniF,Laps, N - Exclude); + _ -> + {skipped, "Too few schedulers online"} end. run_sched_workers(InitF,ExecF,FiniF,Laps) -> @@ -6231,11 +6221,9 @@ spawn_monitor_with_pid(Pid, Fun, N) -> only_if_smp(Func) -> only_if_smp(2, Func). only_if_smp(Schedulers, Func) -> - case {erlang:system_info(smp_support), - erlang:system_info(schedulers_online)} of - {false,_} -> {skip,"No smp support"}; - {true,N} when N < Schedulers -> {skip,"Too few schedulers online"}; - {true,_} -> Func() + case erlang:system_info(schedulers_online) of + N when N < Schedulers -> {skip,"Too few schedulers online"}; + _ -> Func() end. %% Copy-paste from emulator/test/binary_SUITE.erl diff --git a/lib/stdlib/test/proc_lib_SUITE.erl b/lib/stdlib/test/proc_lib_SUITE.erl index 029e6286e4..c4fafe82a4 100644 --- a/lib/stdlib/test/proc_lib_SUITE.erl +++ b/lib/stdlib/test/proc_lib_SUITE.erl @@ -28,7 +28,7 @@ init_per_group/2,end_per_group/2, crash/1, stacktrace/1, sync_start_nolink/1, sync_start_link/1, spawn_opt/1, sp1/0, sp2/0, sp3/1, sp4/2, sp5/1, '\x{447}'/0, - hibernate/1, stop/1, t_format/1]). + hibernate/1, stop/1, t_format/1, t_format_arbitrary/1]). -export([ otp_6345/1, init_dont_hang/1]). -export([hib_loop/1, awaken/1]). @@ -51,7 +51,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [crash, stacktrace, {group, sync_start}, spawn_opt, hibernate, - {group, tickets}, stop, t_format]. + {group, tickets}, stop, t_format, t_format_arbitrary]. groups() -> [{tickets, [], [otp_6345, init_dont_hang]}, @@ -78,6 +78,14 @@ end_per_group(_GroupName, Config) -> %% synchronous, and we want to test that the crash report is ok. %%----------------------------------------------------------------- crash(Config) when is_list(Config) -> + ok = application:unset_env(kernel, error_logger_format_depth), + crash_1(Config), + ok = application:set_env(kernel, error_logger_format_depth, 30), + crash_1(Config), + ok = application:unset_env(kernel, error_logger_format_depth), + ok. + +crash_1(_Config) -> error_logger:add_report_handler(?MODULE, self()), %% Make sure that we don't get a crash report if a process @@ -562,9 +570,32 @@ t_format() -> ok. +t_format_arbitrary(_Config) -> + error_logger:tty(false), + try + t_format_arbitrary() + after + error_logger:tty(true) + end, + ok. + +t_format_arbitrary() -> + A = list_to_atom([1024]), + do_test_format([fake_report, A], unlimited), + do_test_format([fake_report, A], 20), + + do_test_format([fake_report, foo], unlimited), + do_test_format([fake_report, foo], 20), + do_test_format([fake_report, []], unlimited), + do_test_format([fake_report, []], 20). + do_test_format(Report, Depth) -> - io:format("*** Depth = ~p", [Depth]), - S0 = proc_lib:format(Report, latin1, Depth), + do_test_format(Report, latin1, Depth), + do_test_format(Report, unicode, Depth). + +do_test_format(Report, Encoding, Depth) -> + io:format("*** Depth = ~p, Encoding = ~p", [Depth, Encoding]), + S0 = proc_lib:format(Report, Encoding, Depth), S = lists:flatten(S0), io:put_chars(S), length(S). @@ -584,7 +615,7 @@ init(Tester) -> {ok, Tester}. handle_event({error_report, _GL, {Pid, crash_report, Report}}, Tester) -> - io:format("~s\n", [proc_lib:format(Report)]), + io:format("~ts\n", [proc_lib:format(Report)]), Tester ! {crash_report, Pid, Report}, {ok, Tester}; handle_event(_Event, State) -> diff --git a/lib/stdlib/test/re_SUITE_data/testoutput1 b/lib/stdlib/test/re_SUITE_data/testoutput1 index a2b3cffe9d..eff8ecc948 100644 --- a/lib/stdlib/test/re_SUITE_data/testoutput1 +++ b/lib/stdlib/test/re_SUITE_data/testoutput1 @@ -9442,4 +9442,8 @@ No match \ X 0: X +/X+(?#comment)?/ + >XXX< + 0: X + /-- End of testinput1 --/ diff --git a/lib/stdlib/test/re_SUITE_data/testoutput8 b/lib/stdlib/test/re_SUITE_data/testoutput8 index 17b667a980..4984376d3c 100644 --- a/lib/stdlib/test/re_SUITE_data/testoutput8 +++ b/lib/stdlib/test/re_SUITE_data/testoutput8 @@ -7801,4 +7801,8 @@ No match ** Show all captures ignored after DFA matching 0: a +/(02-)?[0-9]{3}-[0-9]{3}/ + 02-123-123 + 0: 02-123-123 + /-- End of testinput8 --/ diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index 4f0fdc4c6a..217e8cc252 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -31,7 +31,7 @@ progex_lc/1, progex_funs/1, otp_5990/1, otp_6166/1, otp_6554/1, otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1, - otp_14285/1, otp_14296/1]). + otp_14285/1, otp_14296/1, typed_records/1]). -export([ start_restricted_from_shell/1, start_restricted_on_command_line/1,restricted_local/1]). @@ -74,10 +74,10 @@ suite() -> {timetrap,{minutes,10}}]. all() -> - [forget, records, known_bugs, otp_5226, otp_5327, + [forget, known_bugs, otp_5226, otp_5327, otp_5435, otp_5195, otp_5915, otp_5916, {group, bits}, {group, refman}, {group, progex}, {group, tickets}, - {group, restricted}]. + {group, restricted}, {group, records}]. groups() -> [{restricted, [], @@ -86,6 +86,8 @@ groups() -> {bits, [], [bs_match_misc_SUITE, bs_match_tail_SUITE, bs_match_bin_SUITE, bs_construct_SUITE]}, + {records, [], + [records, typed_records]}, {refman, [], [refman_bit_syntax]}, {progex, [], [progex_bit_syntax, progex_records, progex_lc, @@ -486,6 +488,48 @@ records(Config) when is_list(Config) -> ok. +%% Test of typed record support. +typed_records(Config) when is_list(Config) -> + Test = filename:join(proplists:get_value(priv_dir, Config), "test.hrl"), + Contents = <<"-module(test). + -record(r0,{f :: any()}). + -record(r1,{f1 :: #r1{} | undefined, f2 :: #r0{} | atom()}). + -record(r2,{f :: #r2{} | undefined}). + ">>, + ok = file:write_file(Test, Contents), + + RR1 = "rr(\"" ++ Test ++ "\"), + #r1{} = (#r1{f1=#r1{f1=undefined, f2=x}, f2 = #r0{}})#r1.f1, + ok.", + RR2 = "rr(\"" ++ Test ++ "\"), + #r0{} = (#r1{f1=#r1{f1=undefined, f2=x}, f2 = #r0{}})#r1.f2, + ok. ", + RR3 = "rr(\"" ++ Test ++ "\"), + #r1{f2=#r0{}} = (#r1{f1=#r1{f1=undefined, f2=#r0{}}, f2 = x})#r1.f1, + ok.", + RR4 = "rr(\"" ++ Test ++ "\"), + (#r1{f2 = #r0{}})#r1{f2 = x}, + ok. ", + RR5 = "rr(\"" ++ Test ++ "\"), + (#r1{f2 = #r0{}})#r1{f1 = #r1{}}, + ok. ", + RR6 = "rr(\"" ++ Test ++ "\"), + (#r2{f=#r2{f=undefined}})#r2.f, + ok.", + RR7 = "rr(\"" ++ Test ++ "\"), + #r2{} = (#r2{f=#r2{f=undefined}})#r2.f, + ok.", + [ok] = scan(RR1), + [ok] = scan(RR2), + [ok] = scan(RR3), + [ok] = scan(RR4), + [ok] = scan(RR5), + [ok] = scan(RR6), + [ok] = scan(RR7), + + file:delete(Test), + ok. + %% Known bugs. known_bugs(Config) when is_list(Config) -> %% erl_eval:merge_bindings/2 cannot handle _removal_ of bindings. diff --git a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt index 4bb4b1369b..d7d8f90de0 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt @@ -1,23 +1,24 @@ -# GraphemeBreakTest-9.0.0.txt -# Date: 2016-06-02, 18:28:17 GMT -# © 2016 Unicode®, Inc. +# GraphemeBreakTest-10.0.0.txt +# Date: 2017-04-14, 05:40:29 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # # Unicode Character Database # For documentation, see http://www.unicode.org/reports/tr44/ # -# Default Grapheme Break Test +# Default Grapheme_Cluster_Break Test # # Format: -# <string> (# <comment>)? -# <string> contains hex Unicode code points, with -# ÷ wherever there is a break opportunity, and +# <string> (# <comment>)? +# <string> contains hex Unicode code points, with +# ÷ wherever there is a break opportunity, and # × wherever there is not. # <comment> the format can change, but currently it shows: # - the sample character name # - (x) the Grapheme_Cluster_Break property value for the sample character -# - [x] the rule that determines whether there is a break or not +# - [x] the rule that determines whether there is a break or not, +# as listed in the Rules section of GraphemeBreakTest.html # # These samples may be extended or changed in the future. # @@ -53,8 +54,8 @@ ÷ 0020 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 0020 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 0020 × 0308 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0020 ÷ 2764 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 0020 × 0308 ÷ 2764 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 0020 ÷ 2640 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 0020 × 0308 ÷ 2640 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 0020 ÷ 1F466 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0020 × 0308 ÷ 1F466 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0020 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -93,8 +94,8 @@ ÷ 000D ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 000D ÷ 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 000D ÷ 0308 × 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 000D ÷ 2764 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 000D ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 000D ÷ 2640 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 000D ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 000D ÷ 1F466 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] BOY (EBG) ÷ [0.3] ÷ 000D ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 000D ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] @@ -133,8 +134,8 @@ ÷ 000A ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 000A ÷ 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 000A ÷ 0308 × 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 000A ÷ 2764 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 000A ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 000A ÷ 2640 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 000A ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 000A ÷ 1F466 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] BOY (EBG) ÷ [0.3] ÷ 000A ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 000A ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] @@ -173,8 +174,8 @@ ÷ 0001 ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 0001 ÷ 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 0001 ÷ 0308 × 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0001 ÷ 2764 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 0001 ÷ 2640 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 0001 ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 0001 ÷ 1F466 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] BOY (EBG) ÷ [0.3] ÷ 0001 ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0001 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] @@ -213,8 +214,8 @@ ÷ 0300 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 0300 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 0300 × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0300 ÷ 2764 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 0300 × 0308 ÷ 2764 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 0300 ÷ 2640 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 0300 × 0308 ÷ 2640 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 0300 ÷ 1F466 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0300 × 0308 ÷ 1F466 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0300 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -253,8 +254,8 @@ ÷ 0600 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 0600 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 0600 × 0308 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0600 × 2764 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 0600 × 0308 ÷ 2764 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 0600 × 2640 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 0600 × 0308 ÷ 2640 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 0600 × 1F466 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] BOY (EBG) ÷ [0.3] ÷ 0600 × 0308 ÷ 1F466 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0600 × 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] <reserved-0378> (Other) ÷ [0.3] @@ -293,8 +294,8 @@ ÷ 0903 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 0903 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 0903 × 0308 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0903 ÷ 2764 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 0903 × 0308 ÷ 2764 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 0903 ÷ 2640 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 0903 × 0308 ÷ 2640 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 0903 ÷ 1F466 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0903 × 0308 ÷ 1F466 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0903 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -333,8 +334,8 @@ ÷ 1100 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 1100 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 1100 × 0308 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1100 ÷ 2764 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 1100 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 1100 ÷ 2640 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 1100 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 1100 ÷ 1F466 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1100 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1100 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -373,8 +374,8 @@ ÷ 1160 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 1160 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 1160 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1160 ÷ 2764 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 1160 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 1160 ÷ 2640 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 1160 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 1160 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1160 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1160 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -413,8 +414,8 @@ ÷ 11A8 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 11A8 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 11A8 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 11A8 ÷ 2764 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 11A8 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 11A8 ÷ 2640 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 11A8 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 11A8 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 11A8 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 11A8 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -453,8 +454,8 @@ ÷ AC00 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ AC00 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ AC00 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ AC00 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ AC00 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ AC00 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ AC00 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ AC00 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ AC00 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ AC00 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -493,8 +494,8 @@ ÷ AC01 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ AC01 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ AC01 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ AC01 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ AC01 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ AC01 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ AC01 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ AC01 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ AC01 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ AC01 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -533,8 +534,8 @@ ÷ 1F1E6 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 1F1E6 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 1F1E6 × 0308 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F1E6 ÷ 2764 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ 2764 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 1F1E6 ÷ 2640 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 1F1E6 × 0308 ÷ 2640 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 1F1E6 ÷ 1F466 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1F1E6 × 0308 ÷ 1F466 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1F1E6 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -573,8 +574,8 @@ ÷ 261D × 0308 × 1F3FB ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 261D × 200D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 261D × 0308 × 200D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 261D ÷ 2764 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 261D × 0308 ÷ 2764 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 261D ÷ 2640 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 261D × 0308 ÷ 2640 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 261D ÷ 1F466 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 261D × 0308 ÷ 1F466 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 261D ÷ 0378 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -613,8 +614,8 @@ ÷ 1F3FB × 0308 ÷ 1F3FB ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 1F3FB × 200D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 1F3FB × 0308 × 200D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F3FB ÷ 2764 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 1F3FB × 0308 ÷ 2764 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 1F3FB ÷ 2640 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 1F3FB × 0308 ÷ 2640 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 1F3FB ÷ 1F466 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1F3FB × 0308 ÷ 1F466 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1F3FB ÷ 0378 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -653,54 +654,54 @@ ÷ 200D × 0308 ÷ 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 200D × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 200D × 0308 × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 200D × 2764 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 200D × 0308 ÷ 2764 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 200D × 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 200D × 0308 ÷ 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 200D × 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) ÷ [0.3] ÷ 200D × 0308 ÷ 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 200D ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 200D × 0308 ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 200D ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 200D × 0308 ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 2764 ÷ 0020 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 2764 × 0308 ÷ 0020 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ 2764 ÷ 000D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 2764 × 0308 ÷ 000D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ 2764 ÷ 000A ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 2764 × 0308 ÷ 000A ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ 2764 ÷ 0001 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 2764 × 0308 ÷ 0001 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ 2764 × 0300 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 2764 × 0308 × 0300 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] -÷ 2764 ÷ 0600 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 2764 × 0308 ÷ 0600 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ 2764 × 0903 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 2764 × 0308 × 0903 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ 2764 ÷ 1100 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 2764 × 0308 ÷ 1100 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ 2764 ÷ 1160 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 2764 × 0308 ÷ 1160 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ 2764 ÷ 11A8 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 2764 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ 2764 ÷ AC00 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 2764 × 0308 ÷ AC00 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ 2764 ÷ AC01 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 2764 × 0308 ÷ AC01 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ 2764 ÷ 1F1E6 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 2764 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ 2764 ÷ 261D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 2764 × 0308 ÷ 261D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] -÷ 2764 ÷ 1F3FB ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 2764 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 2764 × 200D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 2764 × 0308 × 200D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 2764 ÷ 2764 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 2764 × 0308 ÷ 2764 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 2764 ÷ 1F466 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 2764 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] -÷ 2764 ÷ 0378 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 2764 × 0308 ÷ 0378 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 2764 ÷ D800 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 2764 × 0308 ÷ D800 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 2640 ÷ 0020 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 2640 × 0308 ÷ 0020 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] +÷ 2640 ÷ 000D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 2640 × 0308 ÷ 000D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] +÷ 2640 ÷ 000A ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 2640 × 0308 ÷ 000A ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] +÷ 2640 ÷ 0001 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 2640 × 0308 ÷ 0001 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] +÷ 2640 × 0300 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 2640 × 0308 × 0300 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3] +÷ 2640 ÷ 0600 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 2640 × 0308 ÷ 0600 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] +÷ 2640 × 0903 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 2640 × 0308 × 0903 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] +÷ 2640 ÷ 1100 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 2640 × 0308 ÷ 1100 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] +÷ 2640 ÷ 1160 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 2640 × 0308 ÷ 1160 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] +÷ 2640 ÷ 11A8 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 2640 × 0308 ÷ 11A8 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] +÷ 2640 ÷ AC00 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 2640 × 0308 ÷ AC00 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] +÷ 2640 ÷ AC01 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 2640 × 0308 ÷ AC01 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] +÷ 2640 ÷ 1F1E6 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 2640 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] +÷ 2640 ÷ 261D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] +÷ 2640 × 0308 ÷ 261D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] +÷ 2640 ÷ 1F3FB ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] +÷ 2640 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] +÷ 2640 × 200D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] +÷ 2640 × 0308 × 200D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] +÷ 2640 ÷ 2640 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 2640 × 0308 ÷ 2640 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 2640 ÷ 1F466 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 2640 × 0308 ÷ 1F466 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] +÷ 2640 ÷ 0378 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 2640 × 0308 ÷ 0378 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] +÷ 2640 ÷ D800 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] +÷ 2640 × 0308 ÷ D800 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 1F466 ÷ 0020 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1F466 × 0308 ÷ 0020 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1F466 ÷ 000D ÷ # ÷ [0.2] BOY (EBG) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -733,8 +734,8 @@ ÷ 1F466 × 0308 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 1F466 × 200D ÷ # ÷ [0.2] BOY (EBG) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 1F466 × 0308 × 200D ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 1F466 ÷ 2764 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 1F466 × 0308 ÷ 2764 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 1F466 ÷ 2640 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 1F466 × 0308 ÷ 2640 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 1F466 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1F466 × 0308 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 1F466 ÷ 0378 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -773,8 +774,8 @@ ÷ 0378 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 0378 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ 0378 × 0308 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ 0378 ÷ 2764 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ 0378 × 0308 ÷ 2764 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 0378 ÷ 2640 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ 0378 × 0308 ÷ 2640 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 0378 ÷ 1F466 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0378 × 0308 ÷ 1F466 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ 0378 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] @@ -813,8 +814,8 @@ ÷ D800 ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ D800 ÷ 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] ÷ D800 ÷ 0308 × 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3] -÷ D800 ÷ 2764 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ D800 ÷ 2640 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] +÷ D800 ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ D800 ÷ 1F466 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] BOY (EBG) ÷ [0.3] ÷ D800 ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3] ÷ D800 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] @@ -840,7 +841,7 @@ ÷ 261D × 1F3FB ÷ 261D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3] ÷ 1F466 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] ÷ 200D × 1F466 × 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3] -÷ 200D × 2764 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3] +÷ 200D × 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3] ÷ 200D × 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) ÷ [0.3] ÷ 1F466 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] BOY (EBG) ÷ [0.3] # diff --git a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt index 05efcf5a44..6715446aba 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt @@ -1,25 +1,28 @@ -# LineBreakTest-9.0.0.txt -# Date: 2016-06-18, 00:42:06 GMT -# © 2016 Unicode®, Inc. +# LineBreakTest-10.0.0.txt +# Date: 2017-04-14, 05:40:30 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # # Unicode Character Database # For documentation, see http://www.unicode.org/reports/tr44/ # -# Default Line Break Test +# Default Line_Break Test # # Format: -# <string> (# <comment>)? -# <string> contains hex Unicode code points, with -# ÷ wherever there is a break opportunity, and +# <string> (# <comment>)? +# <string> contains hex Unicode code points, with +# ÷ wherever there is a break opportunity, and # × wherever there is not. # <comment> the format can change, but currently it shows: # - the sample character name # - (x) the Line_Break property value for the sample character -# - [x] the rule that determines whether there is a break or not -# Note: The Line Break tests use tailoring of numbers described in Example 7 of Section 8.2 Examples of Customization. -# They also differ from the results produced by a pair table implementation in sequences like: ZW SP CL. +# - [x] the rule that determines whether there is a break or not, +# as listed in the Rules section of LineBreakTest.html +# +# Note: +# The Line_Break tests use tailoring of numbers described in +# Example 7 of Section 8.2, "Examples of Customization" of UAX #14. # # These samples may be extended or changed in the future. # diff --git a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt index e133fa8a78..71f2371c5e 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt @@ -1,6 +1,6 @@ -# NormalizationTest-9.0.0.txt -# Date: 2016-04-04, 11:41:55 GMT -# © 2016 Unicode®, Inc. +# NormalizationTest-10.0.0.txt +# Date: 2017-03-08, 08:41:55 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -17653,6 +17653,10 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 0CBC 3099 093C 0334 0062;0061 0334 0CBC 093C 3099 0062;0061 0334 0CBC 093C 3099 0062;0061 0334 0CBC 093C 3099 0062;0061 0334 0CBC 093C 3099 0062; # (a◌಼◌゙◌़◌̴b; a◌̴◌಼◌़◌゙b; a◌̴◌಼◌़◌゙b; a◌̴◌಼◌़◌゙b; a◌̴◌಼◌़◌゙b; ) LATIN SMALL LETTER A, KANNADA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 0CCD 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062; # (a◌ְ◌्◌゙◌್b; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KANNADA SIGN VIRAMA, LATIN SMALL LETTER B 0061 0CCD 05B0 094D 3099 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062; # (a◌್◌ְ◌्◌゙b; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; ) LATIN SMALL LETTER A, KANNADA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0D3B 0062;0061 3099 094D 0D3B 05B0 0062;0061 3099 094D 0D3B 05B0 0062;0061 3099 094D 0D3B 05B0 0062;0061 3099 094D 0D3B 05B0 0062; # (a◌ְ◌्◌゙◌഻b; a◌゙◌्◌഻◌ְb; a◌゙◌्◌഻◌ְb; a◌゙◌्◌഻◌ְb; a◌゙◌्◌഻◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN VERTICAL BAR VIRAMA, LATIN SMALL LETTER B +0061 0D3B 05B0 094D 3099 0062;0061 3099 0D3B 094D 05B0 0062;0061 3099 0D3B 094D 05B0 0062;0061 3099 0D3B 094D 05B0 0062;0061 3099 0D3B 094D 05B0 0062; # (a◌഻◌ְ◌्◌゙b; a◌゙◌഻◌्◌ְb; a◌゙◌഻◌्◌ְb; a◌゙◌഻◌्◌ְb; a◌゙◌഻◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN VERTICAL BAR VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0D3C 0062;0061 3099 094D 0D3C 05B0 0062;0061 3099 094D 0D3C 05B0 0062;0061 3099 094D 0D3C 05B0 0062;0061 3099 094D 0D3C 05B0 0062; # (a◌ְ◌्◌゙◌഼b; a◌゙◌्◌഼◌ְb; a◌゙◌्◌഼◌ְb; a◌゙◌्◌഼◌ְb; a◌゙◌्◌഼◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN CIRCULAR VIRAMA, LATIN SMALL LETTER B +0061 0D3C 05B0 094D 3099 0062;0061 3099 0D3C 094D 05B0 0062;0061 3099 0D3C 094D 05B0 0062;0061 3099 0D3C 094D 05B0 0062;0061 3099 0D3C 094D 05B0 0062; # (a◌഼◌ְ◌्◌゙b; a◌゙◌഼◌्◌ְb; a◌゙◌഼◌्◌ְb; a◌゙◌഼◌्◌ְb; a◌゙◌഼◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN CIRCULAR VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 0D4D 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062; # (a◌ְ◌्◌゙◌്b; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN VIRAMA, LATIN SMALL LETTER B 0061 0D4D 05B0 094D 3099 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062; # (a◌്◌ְ◌्◌゙b; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 0DCA 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062; # (a◌ְ◌्◌゙◌්b; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, SINHALA SIGN AL-LAKUNA, LATIN SMALL LETTER B @@ -17999,6 +18003,14 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 1DF4 0315 0300 05AE 0062;0061 05AE 1DF4 0300 0315 0062;0061 05AE 1DF4 0300 0315 0062;0061 05AE 1DF4 0300 0315 0062;0061 05AE 1DF4 0300 0315 0062; # (a◌ᷴ◌̕◌̀◌֮b; a◌֮◌ᷴ◌̀◌̕b; a◌֮◌ᷴ◌̀◌̕b; a◌֮◌ᷴ◌̀◌̕b; a◌֮◌ᷴ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER U WITH DIAERESIS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 1DF5 0062;00E0 05AE 1DF5 0315 0062;0061 05AE 0300 1DF5 0315 0062;00E0 05AE 1DF5 0315 0062;0061 05AE 0300 1DF5 0315 0062; # (a◌̕◌̀◌֮◌᷵b; à◌֮◌᷵◌̕b; a◌֮◌̀◌᷵◌̕b; à◌֮◌᷵◌̕b; a◌֮◌̀◌᷵◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING UP TACK ABOVE, LATIN SMALL LETTER B 0061 1DF5 0315 0300 05AE 0062;0061 05AE 1DF5 0300 0315 0062;0061 05AE 1DF5 0300 0315 0062;0061 05AE 1DF5 0300 0315 0062;0061 05AE 1DF5 0300 0315 0062; # (a◌᷵◌̕◌̀◌֮b; a◌֮◌᷵◌̀◌̕b; a◌֮◌᷵◌̀◌̕b; a◌֮◌᷵◌̀◌̕b; a◌֮◌᷵◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING UP TACK ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 035C 0315 0300 1DF6 0062;00E0 0315 1DF6 035C 0062;0061 0300 0315 1DF6 035C 0062;00E0 0315 1DF6 035C 0062;0061 0300 0315 1DF6 035C 0062; # (a◌͜◌̕◌̀◌᷶b; à◌̕◌᷶◌͜b; a◌̀◌̕◌᷶◌͜b; à◌̕◌᷶◌͜b; a◌̀◌̕◌᷶◌͜b; ) LATIN SMALL LETTER A, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, COMBINING KAVYKA ABOVE RIGHT, LATIN SMALL LETTER B +0061 1DF6 035C 0315 0300 0062;00E0 1DF6 0315 035C 0062;0061 0300 1DF6 0315 035C 0062;00E0 1DF6 0315 035C 0062;0061 0300 1DF6 0315 035C 0062; # (a◌᷶◌͜◌̕◌̀b; à◌᷶◌̕◌͜b; a◌̀◌᷶◌̕◌͜b; à◌᷶◌̕◌͜b; a◌̀◌᷶◌̕◌͜b; ) LATIN SMALL LETTER A, COMBINING KAVYKA ABOVE RIGHT, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 0300 05AE 1D16D 1DF7 0062;00E0 1D16D 05AE 1DF7 0062;0061 1D16D 05AE 1DF7 0300 0062;00E0 1D16D 05AE 1DF7 0062;0061 1D16D 05AE 1DF7 0300 0062; # (a◌̀◌𝅭֮◌᷷b; à𝅭◌֮◌᷷b; a𝅭◌֮◌᷷◌̀b; à𝅭◌֮◌᷷b; a𝅭◌֮◌᷷◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, COMBINING KAVYKA ABOVE LEFT, LATIN SMALL LETTER B +0061 1DF7 0300 05AE 1D16D 0062;00E0 1D16D 1DF7 05AE 0062;0061 1D16D 1DF7 05AE 0300 0062;00E0 1D16D 1DF7 05AE 0062;0061 1D16D 1DF7 05AE 0300 0062; # (a◌᷷◌̀◌𝅭֮b; à𝅭◌᷷◌֮b; a𝅭◌᷷◌֮◌̀b; à𝅭◌᷷◌֮b; a𝅭◌᷷◌֮◌̀b; ) LATIN SMALL LETTER A, COMBINING KAVYKA ABOVE LEFT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B +0061 0300 05AE 1D16D 1DF8 0062;00E0 1D16D 05AE 1DF8 0062;0061 1D16D 05AE 1DF8 0300 0062;00E0 1D16D 05AE 1DF8 0062;0061 1D16D 05AE 1DF8 0300 0062; # (a◌̀◌𝅭֮◌᷸b; à𝅭◌֮◌᷸b; a𝅭◌֮◌᷸◌̀b; à𝅭◌֮◌᷸b; a𝅭◌֮◌᷸◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, COMBINING DOT ABOVE LEFT, LATIN SMALL LETTER B +0061 1DF8 0300 05AE 1D16D 0062;00E0 1D16D 1DF8 05AE 0062;0061 1D16D 1DF8 05AE 0300 0062;00E0 1D16D 1DF8 05AE 0062;0061 1D16D 1DF8 05AE 0300 0062; # (a◌᷸◌̀◌𝅭֮b; à𝅭◌᷸◌֮b; a𝅭◌᷸◌֮◌̀b; à𝅭◌᷸◌֮b; a𝅭◌᷸◌֮◌̀b; ) LATIN SMALL LETTER A, COMBINING DOT ABOVE LEFT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B +0061 059A 0316 302A 1DF9 0062;0061 302A 0316 1DF9 059A 0062;0061 302A 0316 1DF9 059A 0062;0061 302A 0316 1DF9 059A 0062;0061 302A 0316 1DF9 059A 0062; # (a◌֚◌̖◌〪◌᷹b; a◌〪◌̖◌᷹◌֚b; a◌〪◌̖◌᷹◌֚b; a◌〪◌̖◌᷹◌֚b; a◌〪◌̖◌᷹◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING WIDE INVERTED BRIDGE BELOW, LATIN SMALL LETTER B +0061 1DF9 059A 0316 302A 0062;0061 302A 1DF9 0316 059A 0062;0061 302A 1DF9 0316 059A 0062;0061 302A 1DF9 0316 059A 0062;0061 302A 1DF9 0316 059A 0062; # (a◌᷹◌֚◌̖◌〪b; a◌〪◌᷹◌̖◌֚b; a◌〪◌᷹◌̖◌֚b; a◌〪◌᷹◌̖◌֚b; a◌〪◌᷹◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING WIDE INVERTED BRIDGE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 0315 0300 05AE 1DFB 0062;00E0 05AE 1DFB 0315 0062;0061 05AE 0300 1DFB 0315 0062;00E0 05AE 1DFB 0315 0062;0061 05AE 0300 1DFB 0315 0062; # (a◌̕◌̀◌֮◌᷻b; à◌֮◌᷻◌̕b; a◌֮◌̀◌᷻◌̕b; à◌֮◌᷻◌̕b; a◌֮◌̀◌᷻◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DELETION MARK, LATIN SMALL LETTER B 0061 1DFB 0315 0300 05AE 0062;0061 05AE 1DFB 0300 0315 0062;0061 05AE 1DFB 0300 0315 0062;0061 05AE 1DFB 0300 0315 0062;0061 05AE 1DFB 0300 0315 0062; # (a◌᷻◌̕◌̀◌֮b; a◌֮◌᷻◌̀◌̕b; a◌֮◌᷻◌̀◌̕b; a◌֮◌᷻◌̀◌̕b; a◌֮◌᷻◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DELETION MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 035D 035C 0315 1DFC 0062;0061 0315 035C 1DFC 035D 0062;0061 0315 035C 1DFC 035D 0062;0061 0315 035C 1DFC 035D 0062;0061 0315 035C 1DFC 035D 0062; # (a◌͝◌͜◌̕◌᷼b; a◌̕◌͜◌᷼◌͝b; a◌̕◌͜◌᷼◌͝b; a◌̕◌͜◌᷼◌͝b; a◌̕◌͜◌᷼◌͝b; ) LATIN SMALL LETTER A, COMBINING DOUBLE BREVE, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING DOUBLE INVERTED BREVE BELOW, LATIN SMALL LETTER B @@ -18397,8 +18409,20 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 116B7 3099 093C 0334 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062; # (a◌𑚷◌゙◌़◌̴b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; ) LATIN SMALL LETTER A, TAKRI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 05B0 094D 3099 1172B 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062; # (a◌ְ◌्◌゙◌𑜫b; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, AHOM SIGN KILLER, LATIN SMALL LETTER B 0061 1172B 05B0 094D 3099 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062; # (a◌𑜫◌ְ◌्◌゙b; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; ) LATIN SMALL LETTER A, AHOM SIGN KILLER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11A34 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062; # (a◌ְ◌्◌゙◌𑨴b; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SIGN VIRAMA, LATIN SMALL LETTER B +0061 11A34 05B0 094D 3099 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062; # (a◌𑨴◌ְ◌्◌゙b; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; ) LATIN SMALL LETTER A, ZANABAZAR SQUARE SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11A47 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062; # (a◌ְ◌्◌゙◌𑩇b; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SUBJOINER, LATIN SMALL LETTER B +0061 11A47 05B0 094D 3099 0062;0061 3099 11A47 094D 05B0 0062;0061 3099 11A47 094D 05B0 0062;0061 3099 11A47 094D 05B0 0062;0061 3099 11A47 094D 05B0 0062; # (a◌𑩇◌ְ◌्◌゙b; a◌゙◌𑩇◌्◌ְb; a◌゙◌𑩇◌्◌ְb; a◌゙◌𑩇◌्◌ְb; a◌゙◌𑩇◌्◌ְb; ) LATIN SMALL LETTER A, ZANABAZAR SQUARE SUBJOINER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11A99 0062;0061 3099 094D 11A99 05B0 0062;0061 3099 094D 11A99 05B0 0062;0061 3099 094D 11A99 05B0 0062;0061 3099 094D 11A99 05B0 0062; # (a◌ְ◌्◌゙◌𑪙b; a◌゙◌्◌𑪙◌ְb; a◌゙◌्◌𑪙◌ְb; a◌゙◌्◌𑪙◌ְb; a◌゙◌्◌𑪙◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, SOYOMBO SUBJOINER, LATIN SMALL LETTER B +0061 11A99 05B0 094D 3099 0062;0061 3099 11A99 094D 05B0 0062;0061 3099 11A99 094D 05B0 0062;0061 3099 11A99 094D 05B0 0062;0061 3099 11A99 094D 05B0 0062; # (a◌𑪙◌ְ◌्◌゙b; a◌゙◌𑪙◌्◌ְb; a◌゙◌𑪙◌्◌ְb; a◌゙◌𑪙◌्◌ְb; a◌゙◌𑪙◌्◌ְb; ) LATIN SMALL LETTER A, SOYOMBO SUBJOINER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11C3F 0062;0061 3099 094D 11C3F 05B0 0062;0061 3099 094D 11C3F 05B0 0062;0061 3099 094D 11C3F 05B0 0062;0061 3099 094D 11C3F 05B0 0062; # (a◌ְ◌्◌゙◌𑰿b; a◌゙◌्◌𑰿◌ְb; a◌゙◌्◌𑰿◌ְb; a◌゙◌्◌𑰿◌ְb; a◌゙◌्◌𑰿◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, BHAIKSUKI SIGN VIRAMA, LATIN SMALL LETTER B 0061 11C3F 05B0 094D 3099 0062;0061 3099 11C3F 094D 05B0 0062;0061 3099 11C3F 094D 05B0 0062;0061 3099 11C3F 094D 05B0 0062;0061 3099 11C3F 094D 05B0 0062; # (a◌𑰿◌ְ◌्◌゙b; a◌゙◌𑰿◌्◌ְb; a◌゙◌𑰿◌्◌ְb; a◌゙◌𑰿◌्◌ְb; a◌゙◌𑰿◌्◌ְb; ) LATIN SMALL LETTER A, BHAIKSUKI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 11D42 0062;0061 0334 093C 11D42 3099 0062;0061 0334 093C 11D42 3099 0062;0061 0334 093C 11D42 3099 0062;0061 0334 093C 11D42 3099 0062; # (a◌゙◌़◌̴◌𑵂b; a◌̴◌़◌𑵂◌゙b; a◌̴◌़◌𑵂◌゙b; a◌̴◌़◌𑵂◌゙b; a◌̴◌़◌𑵂◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, MASARAM GONDI SIGN NUKTA, LATIN SMALL LETTER B +0061 11D42 3099 093C 0334 0062;0061 0334 11D42 093C 3099 0062;0061 0334 11D42 093C 3099 0062;0061 0334 11D42 093C 3099 0062;0061 0334 11D42 093C 3099 0062; # (a◌𑵂◌゙◌़◌̴b; a◌̴◌𑵂◌़◌゙b; a◌̴◌𑵂◌़◌゙b; a◌̴◌𑵂◌़◌゙b; a◌̴◌𑵂◌़◌゙b; ) LATIN SMALL LETTER A, MASARAM GONDI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 11D44 0062;0061 3099 094D 11D44 05B0 0062;0061 3099 094D 11D44 05B0 0062;0061 3099 094D 11D44 05B0 0062;0061 3099 094D 11D44 05B0 0062; # (a◌ְ◌्◌゙◌𑵄b; a◌゙◌्◌𑵄◌ְb; a◌゙◌्◌𑵄◌ְb; a◌゙◌्◌𑵄◌ְb; a◌゙◌्◌𑵄◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MASARAM GONDI SIGN HALANTA, LATIN SMALL LETTER B +0061 11D44 05B0 094D 3099 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062; # (a◌𑵄◌ְ◌्◌゙b; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; ) LATIN SMALL LETTER A, MASARAM GONDI SIGN HALANTA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11D45 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062; # (a◌ְ◌्◌゙◌𑵅b; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MASARAM GONDI VIRAMA, LATIN SMALL LETTER B +0061 11D45 05B0 094D 3099 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062; # (a◌𑵅◌ְ◌्◌゙b; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; ) LATIN SMALL LETTER A, MASARAM GONDI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 093C 0334 16AF0 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062; # (a◌़◌̴◌𖫰b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING HIGH TONE, LATIN SMALL LETTER B 0061 16AF0 093C 0334 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062; # (a◌𖫰◌़◌̴b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; ) LATIN SMALL LETTER A, BASSA VAH COMBINING HIGH TONE, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 093C 0334 16AF1 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062; # (a◌़◌̴◌𖫱b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING LOW TONE, LATIN SMALL LETTER B diff --git a/lib/stdlib/uc_spec/CaseFolding.txt b/lib/stdlib/uc_spec/CaseFolding.txt index 372ee68bd8..efdf18e441 100644 --- a/lib/stdlib/uc_spec/CaseFolding.txt +++ b/lib/stdlib/uc_spec/CaseFolding.txt @@ -1,6 +1,6 @@ -# CaseFolding-9.0.0.txt -# Date: 2016-03-02, 18:54:54 GMT -# © 2016 Unicode®, Inc. +# CaseFolding-10.0.0.txt +# Date: 2017-04-14, 05:40:18 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -24,7 +24,7 @@ # # NOTE: case folding does not preserve normalization formats! # -# For information on case folding, including how to have case folding +# For information on case folding, including how to have case folding # preserve normalization formats, see Section 3.13 Default Case Algorithms in # The Unicode Standard. # diff --git a/lib/stdlib/uc_spec/CompositionExclusions.txt b/lib/stdlib/uc_spec/CompositionExclusions.txt index 1999ed1328..ff42508686 100644 --- a/lib/stdlib/uc_spec/CompositionExclusions.txt +++ b/lib/stdlib/uc_spec/CompositionExclusions.txt @@ -1,6 +1,6 @@ -# CompositionExclusions-9.0.0.txt -# Date: 2016-01-21, 22:00:00 GMT [KW, LI] -# © 2016 Unicode®, Inc. +# CompositionExclusions-10.0.0.txt +# Date: 2017-02-15, 00:00:00 GMT [KW, LI] +# © 2017 Unicode®, Inc. # For terms of use, see http://www.unicode.org/terms_of_use.html # # Unicode Character Database diff --git a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt index c5e94a3762..32bb12e47e 100644 --- a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt +++ b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt @@ -1,6 +1,6 @@ -# GraphemeBreakProperty-9.0.0.txt -# Date: 2016-06-03, 22:23:55 GMT -# © 2016 Unicode®, Inc. +# GraphemeBreakProperty-10.0.0.txt +# Date: 2017-03-12, 07:03:41 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -25,8 +25,11 @@ 0D4E ; Prepend # Lo MALAYALAM LETTER DOT REPH 110BD ; Prepend # Cf KAITHI NUMBER SIGN 111C2..111C3 ; Prepend # Lo [2] SHARADA SIGN JIHVAMULIYA..SHARADA SIGN UPADHMANIYA +11A3A ; Prepend # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA +11A86..11A89 ; Prepend # Lo [4] SOYOMBO CLUSTER-INITIAL LETTER RA..SOYOMBO CLUSTER-INITIAL LETTER SA +11D46 ; Prepend # Lo MASARAM GONDI REPHA -# Total code points: 13 +# Total code points: 19 # ================================================ @@ -126,6 +129,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 0AC7..0AC8 ; Extend # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI 0ACD ; Extend # Mn GUJARATI SIGN VIRAMA 0AE2..0AE3 ; Extend # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AFA..0AFF ; Extend # Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE 0B01 ; Extend # Mn ORIYA SIGN CANDRABINDU 0B3C ; Extend # Mn ORIYA SIGN NUKTA 0B3E ; Extend # Mc ORIYA VOWEL SIGN AA @@ -154,7 +158,8 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 0CCC..0CCD ; Extend # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA 0CD5..0CD6 ; Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK 0CE2..0CE3 ; Extend # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL -0D01 ; Extend # Mn MALAYALAM SIGN CANDRABINDU +0D00..0D01 ; Extend # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU +0D3B..0D3C ; Extend # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA 0D3E ; Extend # Mc MALAYALAM VOWEL SIGN AA 0D41..0D44 ; Extend # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR 0D4D ; Extend # Mn MALAYALAM SIGN VIRAMA @@ -243,7 +248,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 1CED ; Extend # Mn VEDIC SIGN TIRYAK 1CF4 ; Extend # Mn VEDIC TONE CANDRA ABOVE 1CF8..1CF9 ; Extend # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE -1DC0..1DF5 ; Extend # Mn [54] COMBINING DOTTED GRAVE ACCENT..COMBINING UP TACK ABOVE +1DC0..1DF9 ; Extend # Mn [58] COMBINING DOTTED GRAVE ACCENT..COMBINING WIDE INVERTED BRIDGE BELOW 1DFB..1DFF ; Extend # Mn [5] COMBINING DELETION MARK..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW 200C ; Extend # Cf ZERO WIDTH NON-JOINER 20D0..20DC ; Extend # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE @@ -353,6 +358,15 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 1171D..1171F ; Extend # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA 11722..11725 ; Extend # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU 11727..1172B ; Extend # Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER +11A01..11A06 ; Extend # Mn [6] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL SIGN O +11A09..11A0A ; Extend # Mn [2] ZANABAZAR SQUARE VOWEL SIGN REVERSED I..ZANABAZAR SQUARE VOWEL LENGTH MARK +11A33..11A38 ; Extend # Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA +11A3B..11A3E ; Extend # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A47 ; Extend # Mn ZANABAZAR SQUARE SUBJOINER +11A51..11A56 ; Extend # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE +11A59..11A5B ; Extend # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK +11A8A..11A96 ; Extend # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA +11A98..11A99 ; Extend # Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER 11C30..11C36 ; Extend # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L 11C38..11C3D ; Extend # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA 11C3F ; Extend # Mn BHAIKSUKI SIGN VIRAMA @@ -360,6 +374,11 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11CAA..11CB0 ; Extend # Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA 11CB2..11CB3 ; Extend # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E 11CB5..11CB6 ; Extend # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU +11D31..11D36 ; Extend # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A ; Extend # Mn MASARAM GONDI VOWEL SIGN E +11D3C..11D3D ; Extend # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D45 ; Extend # Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA +11D47 ; Extend # Mn MASARAM GONDI RA-KARA 16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F8F..16F92 ; Extend # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW @@ -387,7 +406,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT E0020..E007F ; Extend # Cf [96] TAG SPACE..CANCEL TAG E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -# Total code points: 1828 +# Total code points: 1901 # ================================================ @@ -472,6 +491,7 @@ E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 1C34..1C35 ; SpacingMark # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG 1CE1 ; SpacingMark # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA 1CF2..1CF3 ; SpacingMark # Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIGN ROTATED ARDHAVISARGA +1CF7 ; SpacingMark # Mc VEDIC SIGN ATIKRAMA A823..A824 ; SpacingMark # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I A827 ; SpacingMark # Mc SYLOTI NAGRI VOWEL SIGN OO A880..A881 ; SpacingMark # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA @@ -529,6 +549,10 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 116B6 ; SpacingMark # Mc TAKRI SIGN VIRAMA 11720..11721 ; SpacingMark # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA 11726 ; SpacingMark # Mc AHOM VOWEL SIGN E +11A07..11A08 ; SpacingMark # Mc [2] ZANABAZAR SQUARE VOWEL SIGN AI..ZANABAZAR SQUARE VOWEL SIGN AU +11A39 ; SpacingMark # Mc ZANABAZAR SQUARE SIGN VISARGA +11A57..11A58 ; SpacingMark # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU +11A97 ; SpacingMark # Mc SOYOMBO SIGN VISARGA 11C2F ; SpacingMark # Mc BHAIKSUKI VOWEL SIGN AA 11C3E ; SpacingMark # Mc BHAIKSUKI SIGN VISARGA 11CA9 ; SpacingMark # Mc MARCHEN SUBJOINED LETTER YA @@ -538,7 +562,7 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM 1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT -# Total code points: 341 +# Total code points: 348 # ================================================ @@ -1375,8 +1399,9 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH 26F9 ; E_Base # So PERSON WITH BALL 270A..270D ; E_Base # So [4] RAISED FIST..WRITING HAND 1F385 ; E_Base # So FATHER CHRISTMAS -1F3C3..1F3C4 ; E_Base # So [2] RUNNER..SURFER -1F3CA..1F3CB ; E_Base # So [2] SWIMMER..WEIGHT LIFTER +1F3C2..1F3C4 ; E_Base # So [3] SNOWBOARDER..SURFER +1F3C7 ; E_Base # So HORSE RACING +1F3CA..1F3CC ; E_Base # So [3] SWIMMER..GOLFER 1F442..1F443 ; E_Base # So [2] EAR..NOSE 1F446..1F450 ; E_Base # So [11] WHITE UP POINTING BACKHAND INDEX..OPEN HANDS SIGN 1F46E ; E_Base # So POLICE OFFICER @@ -1385,7 +1410,7 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH 1F481..1F483 ; E_Base # So [3] INFORMATION DESK PERSON..DANCER 1F485..1F487 ; E_Base # So [3] NAIL POLISH..HAIRCUT 1F4AA ; E_Base # So FLEXED BICEPS -1F575 ; E_Base # So SLEUTH OR SPY +1F574..1F575 ; E_Base # So [2] MAN IN BUSINESS SUIT LEVITATING..SLEUTH OR SPY 1F57A ; E_Base # So MAN DANCING 1F590 ; E_Base # So RAISED HAND WITH FINGERS SPLAYED 1F595..1F596 ; E_Base # So [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS @@ -1394,13 +1419,15 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH 1F6A3 ; E_Base # So ROWBOAT 1F6B4..1F6B6 ; E_Base # So [3] BICYCLIST..PEDESTRIAN 1F6C0 ; E_Base # So BATH -1F918..1F91E ; E_Base # So [7] SIGN OF THE HORNS..HAND WITH INDEX AND MIDDLE FINGERS CROSSED +1F6CC ; E_Base # So SLEEPING ACCOMMODATION +1F918..1F91C ; E_Base # So [5] SIGN OF THE HORNS..RIGHT-FACING FIST +1F91E..1F91F ; E_Base # So [2] HAND WITH INDEX AND MIDDLE FINGERS CROSSED..I LOVE YOU HAND SIGN 1F926 ; E_Base # So FACE PALM -1F930 ; E_Base # So PREGNANT WOMAN -1F933..1F939 ; E_Base # So [7] SELFIE..JUGGLING -1F93C..1F93E ; E_Base # So [3] WRESTLERS..HANDBALL +1F930..1F939 ; E_Base # So [10] PREGNANT WOMAN..JUGGLING +1F93D..1F93E ; E_Base # So [2] WATER POLO..HANDBALL +1F9D1..1F9DD ; E_Base # So [13] ADULT..ELF -# Total code points: 79 +# Total code points: 98 # ================================================ @@ -1416,11 +1443,28 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH # ================================================ +2640 ; Glue_After_Zwj # So FEMALE SIGN +2642 ; Glue_After_Zwj # So MALE SIGN +2695..2696 ; Glue_After_Zwj # So [2] STAFF OF AESCULAPIUS..SCALES +2708 ; Glue_After_Zwj # So AIRPLANE 2764 ; Glue_After_Zwj # So HEAVY BLACK HEART +1F308 ; Glue_After_Zwj # So RAINBOW +1F33E ; Glue_After_Zwj # So EAR OF RICE +1F373 ; Glue_After_Zwj # So COOKING +1F393 ; Glue_After_Zwj # So GRADUATION CAP +1F3A4 ; Glue_After_Zwj # So MICROPHONE +1F3A8 ; Glue_After_Zwj # So ARTIST PALETTE +1F3EB ; Glue_After_Zwj # So SCHOOL +1F3ED ; Glue_After_Zwj # So FACTORY 1F48B ; Glue_After_Zwj # So KISS MARK +1F4BB..1F4BC ; Glue_After_Zwj # So [2] PERSONAL COMPUTER..BRIEFCASE +1F527 ; Glue_After_Zwj # So WRENCH +1F52C ; Glue_After_Zwj # So MICROSCOPE 1F5E8 ; Glue_After_Zwj # So LEFT SPEECH BUBBLE +1F680 ; Glue_After_Zwj # So ROCKET +1F692 ; Glue_After_Zwj # So FIRE ENGINE -# Total code points: 3 +# Total code points: 22 # ================================================ diff --git a/lib/stdlib/uc_spec/PropList.txt b/lib/stdlib/uc_spec/PropList.txt index a8c0da7135..9a2d0e4b1c 100644 --- a/lib/stdlib/uc_spec/PropList.txt +++ b/lib/stdlib/uc_spec/PropList.txt @@ -1,6 +1,6 @@ -# PropList-9.0.0.txt -# Date: 2016-06-01, 10:34:30 GMT -# © 2016 Unicode®, Inc. +# PropList-10.0.0.txt +# Date: 2017-03-10, 08:25:30 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -199,6 +199,9 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA 115C9..115D7 ; Terminal_Punctuation # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES 11641..11642 ; Terminal_Punctuation # Po [2] MODI DANDA..MODI DOUBLE DANDA 1173C..1173E ; Terminal_Punctuation # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +11A42..11A43 ; Terminal_Punctuation # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD +11A9B..11A9C ; Terminal_Punctuation # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD +11AA1..11AA2 ; Terminal_Punctuation # Po [2] SOYOMBO TERMINAL MARK-1..SOYOMBO TERMINAL MARK-2 11C41..11C43 ; Terminal_Punctuation # Po [3] BHAIKSUKI DANDA..BHAIKSUKI WORD SEPARATOR 11C71 ; Terminal_Punctuation # Po MARCHEN MARK SHAD 12470..12474 ; Terminal_Punctuation # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON @@ -209,7 +212,7 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA 1BC9F ; Terminal_Punctuation # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP 1DA87..1DA8A ; Terminal_Punctuation # Po [4] SIGNWRITING COMMA..SIGNWRITING COLON -# Total code points: 246 +# Total code points: 252 # ================================================ @@ -471,6 +474,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 0AC9 ; Other_Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O 0ACB..0ACC ; Other_Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU 0AE2..0AE3 ; Other_Alphabetic # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AFA..0AFC ; Other_Alphabetic # Mn [3] GUJARATI SIGN SUKUN..GUJARATI SIGN MADDAH 0B01 ; Other_Alphabetic # Mn ORIYA SIGN CANDRABINDU 0B02..0B03 ; Other_Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA 0B3E ; Other_Alphabetic # Mc ORIYA VOWEL SIGN AA @@ -508,7 +512,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 0CCC ; Other_Alphabetic # Mn KANNADA VOWEL SIGN AU 0CD5..0CD6 ; Other_Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK 0CE2..0CE3 ; Other_Alphabetic # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL -0D01 ; Other_Alphabetic # Mn MALAYALAM SIGN CANDRABINDU +0D00..0D01 ; Other_Alphabetic # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU 0D02..0D03 ; Other_Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA 0D3E..0D40 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II 0D41..0D44 ; Other_Alphabetic # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR @@ -726,6 +730,17 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11722..11725 ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU 11726 ; Other_Alphabetic # Mc AHOM VOWEL SIGN E 11727..1172A ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN AW..AHOM VOWEL SIGN AM +11A01..11A06 ; Other_Alphabetic # Mn [6] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL SIGN O +11A07..11A08 ; Other_Alphabetic # Mc [2] ZANABAZAR SQUARE VOWEL SIGN AI..ZANABAZAR SQUARE VOWEL SIGN AU +11A09..11A0A ; Other_Alphabetic # Mn [2] ZANABAZAR SQUARE VOWEL SIGN REVERSED I..ZANABAZAR SQUARE VOWEL LENGTH MARK +11A35..11A38 ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE SIGN CANDRABINDU..ZANABAZAR SQUARE SIGN ANUSVARA +11A39 ; Other_Alphabetic # Mc ZANABAZAR SQUARE SIGN VISARGA +11A3B..11A3E ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A51..11A56 ; Other_Alphabetic # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE +11A57..11A58 ; Other_Alphabetic # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU +11A59..11A5B ; Other_Alphabetic # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK +11A8A..11A96 ; Other_Alphabetic # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA +11A97 ; Other_Alphabetic # Mc SOYOMBO SIGN VISARGA 11C2F ; Other_Alphabetic # Mc BHAIKSUKI VOWEL SIGN AA 11C30..11C36 ; Other_Alphabetic # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L 11C38..11C3D ; Other_Alphabetic # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA @@ -737,6 +752,12 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11CB2..11CB3 ; Other_Alphabetic # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E 11CB4 ; Other_Alphabetic # Mc MARCHEN VOWEL SIGN O 11CB5..11CB6 ; Other_Alphabetic # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU +11D31..11D36 ; Other_Alphabetic # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A ; Other_Alphabetic # Mn MASARAM GONDI VOWEL SIGN E +11D3C..11D3D ; Other_Alphabetic # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D41 ; Other_Alphabetic # Mn [3] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI SIGN VISARGA +11D43 ; Other_Alphabetic # Mn MASARAM GONDI SIGN CANDRA +11D47 ; Other_Alphabetic # Mn MASARAM GONDI RA-KARA 16B30..16B36 ; Other_Alphabetic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F51..16F7E ; Other_Alphabetic # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG 1BC9E ; Other_Alphabetic # Mn DUPLOYAN DOUBLE MARK @@ -750,7 +771,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z 1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z -# Total code points: 1238 +# Total code points: 1300 # ================================================ @@ -759,18 +780,20 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 3021..3029 ; Ideographic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE 3038..303A ; Ideographic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY 3400..4DB5 ; Ideographic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 -4E00..9FD5 ; Ideographic # Lo [20950] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FD5 +4E00..9FEA ; Ideographic # Lo [20971] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEA F900..FA6D ; Ideographic # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 17000..187EC ; Ideographic # Lo [6125] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187EC 18800..18AF2 ; Ideographic # Lo [755] TANGUT COMPONENT-001..TANGUT COMPONENT-755 +1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB 20000..2A6D6 ; Ideographic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 2A700..2B734 ; Ideographic # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 2B740..2B81D ; Ideographic # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D 2B820..2CEA1 ; Ideographic # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D -# Total code points: 88284 +# Total code points: 96174 # ================================================ @@ -826,12 +849,14 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 0A4D ; Diacritic # Mn GURMUKHI SIGN VIRAMA 0ABC ; Diacritic # Mn GUJARATI SIGN NUKTA 0ACD ; Diacritic # Mn GUJARATI SIGN VIRAMA +0AFD..0AFF ; Diacritic # Mn [3] GUJARATI SIGN THREE-DOT NUKTA ABOVE..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE 0B3C ; Diacritic # Mn ORIYA SIGN NUKTA 0B4D ; Diacritic # Mn ORIYA SIGN VIRAMA 0BCD ; Diacritic # Mn TAMIL SIGN VIRAMA 0C4D ; Diacritic # Mn TELUGU SIGN VIRAMA 0CBC ; Diacritic # Mn KANNADA SIGN NUKTA 0CCD ; Diacritic # Mn KANNADA SIGN VIRAMA +0D3B..0D3C ; Diacritic # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA 0D4D ; Diacritic # Mn MALAYALAM SIGN VIRAMA 0DCA ; Diacritic # Mn SINHALA SIGN AL-LAKUNA 0E47..0E4C ; Diacritic # Mn [6] THAI CHARACTER MAITAIKHU..THAI CHARACTER THANTHAKHAT @@ -871,10 +896,11 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 1CE2..1CE8 ; Diacritic # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL 1CED ; Diacritic # Mn VEDIC SIGN TIRYAK 1CF4 ; Diacritic # Mn VEDIC TONE CANDRA ABOVE +1CF7 ; Diacritic # Mc VEDIC SIGN ATIKRAMA 1CF8..1CF9 ; Diacritic # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE 1D2C..1D6A ; Diacritic # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI 1DC4..1DCF ; Diacritic # Mn [12] COMBINING MACRON-ACUTE..COMBINING ZIGZAG BELOW -1DF5 ; Diacritic # Mn COMBINING UP TACK ABOVE +1DF5..1DF9 ; Diacritic # Mn [5] COMBINING UP TACK ABOVE..COMBINING WIDE INVERTED BRIDGE BELOW 1DFD..1DFF ; Diacritic # Mn [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW 1FBD ; Diacritic # Sk GREEK KORONIS 1FBF..1FC1 ; Diacritic # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI @@ -947,7 +973,12 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 116B6 ; Diacritic # Mc TAKRI SIGN VIRAMA 116B7 ; Diacritic # Mn TAKRI SIGN NUKTA 1172B ; Diacritic # Mn AHOM SIGN KILLER +11A34 ; Diacritic # Mn ZANABAZAR SQUARE SIGN VIRAMA +11A47 ; Diacritic # Mn ZANABAZAR SQUARE SUBJOINER +11A99 ; Diacritic # Mn SOYOMBO SUBJOINER 11C3F ; Diacritic # Mn BHAIKSUKI SIGN VIRAMA +11D42 ; Diacritic # Mn MASARAM GONDI SIGN NUKTA +11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA 16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW 16F93..16F9F ; Diacritic # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 @@ -960,7 +991,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK 1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA -# Total code points: 782 +# Total code points: 798 # ================================================ @@ -989,11 +1020,12 @@ AAF3..AAF4 ; Extender # Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETE FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK 1135D ; Extender # Lo GRANTHA SIGN PLUTA 115C6..115C8 ; Extender # Po [3] SIDDHAM REPETITION MARK-1..SIDDHAM REPETITION MARK-3 +11A98 ; Extender # Mn SOYOMBO GEMINATION MARK 16B42..16B43 ; Extender # Lm [2] PAHAWH HMONG SIGN VOS NRUA..PAHAWH HMONG SIGN IB YAM -16FE0 ; Extender # Lm TANGUT ITERATION MARK +16FE0..16FE1 ; Extender # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK 1E944..1E946 ; Extender # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK -# Total code points: 42 +# Total code points: 44 # ================================================ @@ -1105,7 +1137,7 @@ E0020..E007F ; Other_Grapheme_Extend # Cf [96] TAG SPACE..CANCEL TAG # ================================================ 3400..4DB5 ; Unified_Ideograph # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 -4E00..9FD5 ; Unified_Ideograph # Lo [20950] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FD5 +4E00..9FEA ; Unified_Ideograph # Lo [20971] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEA FA0E..FA0F ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F FA11 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA11 FA13..FA14 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14 @@ -1117,8 +1149,9 @@ FA27..FA29 ; Unified_Ideograph # Lo [3] CJK COMPATIBILITY IDEOGRAPH-FA27..C 2A700..2B734 ; Unified_Ideograph # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 2B740..2B81D ; Unified_Ideograph # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D 2B820..2CEA1 ; Unified_Ideograph # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEB0..2EBE0 ; Unified_Ideograph # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 -# Total code points: 80388 +# Total code points: 87882 # ================================================ @@ -1277,6 +1310,8 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP 115C9..115D7 ; Sentence_Terminal # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES 11641..11642 ; Sentence_Terminal # Po [2] MODI DANDA..MODI DOUBLE DANDA 1173C..1173E ; Sentence_Terminal # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +11A42..11A43 ; Sentence_Terminal # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD +11A9B..11A9C ; Sentence_Terminal # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD 11C41..11C42 ; Sentence_Terminal # Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA 16A6E..16A6F ; Sentence_Terminal # Po [2] MRO DANDA..MRO DOUBLE DANDA 16AF5 ; Sentence_Terminal # Po BASSA VAH FULL STOP @@ -1285,7 +1320,7 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP 1BC9F ; Sentence_Terminal # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP 1DA88 ; Sentence_Terminal # Po SIGNWRITING FULL STOP -# Total code points: 124 +# Total code points: 128 # ================================================ @@ -1402,9 +1437,7 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 239B..23B3 ; Pattern_Syntax # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM 23B4..23DB ; Pattern_Syntax # So [40] TOP SQUARE BRACKET..FUSE 23DC..23E1 ; Pattern_Syntax # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET -23E2..23FE ; Pattern_Syntax # So [29] WHITE TRAPEZIUM..POWER SLEEP SYMBOL -23FF ; Pattern_Syntax # Cn <reserved-23FF> -2400..2426 ; Pattern_Syntax # So [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO +23E2..2426 ; Pattern_Syntax # So [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO 2427..243F ; Pattern_Syntax # Cn [25] <reserved-2427>..<reserved-243F> 2440..244A ; Pattern_Syntax # So [11] OCR HOOK..OCR DOUBLE BACKSLASH 244B..245F ; Pattern_Syntax # Cn [21] <reserved-244B>..<reserved-245F> @@ -1492,8 +1525,8 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 2BBA..2BBC ; Pattern_Syntax # Cn [3] <reserved-2BBA>..<reserved-2BBC> 2BBD..2BC8 ; Pattern_Syntax # So [12] BALLOT BOX WITH LIGHT X..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED 2BC9 ; Pattern_Syntax # Cn <reserved-2BC9> -2BCA..2BD1 ; Pattern_Syntax # So [8] TOP HALF BLACK CIRCLE..UNCERTAINTY SIGN -2BD2..2BEB ; Pattern_Syntax # Cn [26] <reserved-2BD2>..<reserved-2BEB> +2BCA..2BD2 ; Pattern_Syntax # So [9] TOP HALF BLACK CIRCLE..GROUP MARK +2BD3..2BEB ; Pattern_Syntax # Cn [25] <reserved-2BD3>..<reserved-2BEB> 2BEC..2BEF ; Pattern_Syntax # So [4] LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS..DOWNWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS 2BF0..2BFF ; Pattern_Syntax # Cn [16] <reserved-2BF0>..<reserved-2BFF> 2E00..2E01 ; Pattern_Syntax # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER @@ -1533,8 +1566,8 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 2E40 ; Pattern_Syntax # Pd DOUBLE HYPHEN 2E41 ; Pattern_Syntax # Po REVERSED COMMA 2E42 ; Pattern_Syntax # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK -2E43..2E44 ; Pattern_Syntax # Po [2] DASH WITH LEFT UPTURN..DOUBLE SUSPENSION MARK -2E45..2E7F ; Pattern_Syntax # Cn [59] <reserved-2E45>..<reserved-2E7F> +2E43..2E49 ; Pattern_Syntax # Po [7] DASH WITH LEFT UPTURN..DOUBLE STACKED COMMA +2E4A..2E7F ; Pattern_Syntax # Cn [54] <reserved-2E4A>..<reserved-2E7F> 3001..3003 ; Pattern_Syntax # Po [3] IDEOGRAPHIC COMMA..DITTO MARK 3008 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET 3009 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET @@ -1576,4 +1609,10 @@ FE45..FE46 ; Pattern_Syntax # Po [2] SESAME DOT..WHITE SESAME DOT # Total code points: 10 +# ================================================ + +1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + +# Total code points: 26 + # EOF diff --git a/lib/stdlib/uc_spec/SpecialCasing.txt b/lib/stdlib/uc_spec/SpecialCasing.txt index b23fa7f768..b9ba0d81c1 100644 --- a/lib/stdlib/uc_spec/SpecialCasing.txt +++ b/lib/stdlib/uc_spec/SpecialCasing.txt @@ -1,6 +1,6 @@ -# SpecialCasing-9.0.0.txt -# Date: 2016-03-02, 18:55:13 GMT -# © 2016 Unicode®, Inc. +# SpecialCasing-10.0.0.txt +# Date: 2017-04-14, 05:40:43 GMT +# © 2017 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -197,7 +197,7 @@ FB17; FB17; 0544 056D; 0544 053D; # ARMENIAN SMALL LIGATURE MEN XEH # ================================================================================ # Conditional Mappings -# The remainder of this file provides conditional casing data used to produce +# The remainder of this file provides conditional casing data used to produce # full case mappings. # ================================================================================ # Language-Insensitive Mappings diff --git a/lib/stdlib/uc_spec/UnicodeData.txt b/lib/stdlib/uc_spec/UnicodeData.txt index a756976461..d89c64f526 100644 --- a/lib/stdlib/uc_spec/UnicodeData.txt +++ b/lib/stdlib/uc_spec/UnicodeData.txt @@ -2072,6 +2072,17 @@ 085A;MANDAIC VOCALIZATION MARK;Mn;220;NSM;;;;;N;;;;; 085B;MANDAIC GEMINATION MARK;Mn;220;NSM;;;;;N;;;;; 085E;MANDAIC PUNCTUATION;Po;0;R;;;;;N;;;;; +0860;SYRIAC LETTER MALAYALAM NGA;Lo;0;AL;;;;;N;;;;; +0861;SYRIAC LETTER MALAYALAM JA;Lo;0;AL;;;;;N;;;;; +0862;SYRIAC LETTER MALAYALAM NYA;Lo;0;AL;;;;;N;;;;; +0863;SYRIAC LETTER MALAYALAM TTA;Lo;0;AL;;;;;N;;;;; +0864;SYRIAC LETTER MALAYALAM NNA;Lo;0;AL;;;;;N;;;;; +0865;SYRIAC LETTER MALAYALAM NNNA;Lo;0;AL;;;;;N;;;;; +0866;SYRIAC LETTER MALAYALAM BHA;Lo;0;AL;;;;;N;;;;; +0867;SYRIAC LETTER MALAYALAM RA;Lo;0;AL;;;;;N;;;;; +0868;SYRIAC LETTER MALAYALAM LLA;Lo;0;AL;;;;;N;;;;; +0869;SYRIAC LETTER MALAYALAM LLLA;Lo;0;AL;;;;;N;;;;; +086A;SYRIAC LETTER MALAYALAM SSA;Lo;0;AL;;;;;N;;;;; 08A0;ARABIC LETTER BEH WITH SMALL V BELOW;Lo;0;AL;;;;;N;;;;; 08A1;ARABIC LETTER BEH WITH HAMZA ABOVE;Lo;0;AL;;;;;N;;;;; 08A2;ARABIC LETTER JEEM WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;; @@ -2366,6 +2377,8 @@ 09F9;BENGALI CURRENCY DENOMINATOR SIXTEEN;No;0;L;;;;16;N;;;;; 09FA;BENGALI ISSHAR;So;0;L;;;;;N;;;;; 09FB;BENGALI GANDA MARK;Sc;0;ET;;;;;N;;;;; +09FC;BENGALI LETTER VEDIC ANUSVARA;Lo;0;L;;;;;N;;;;; +09FD;BENGALI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 0A01;GURMUKHI SIGN ADAK BINDI;Mn;0;NSM;;;;;N;;;;; 0A02;GURMUKHI SIGN BINDI;Mn;0;NSM;;;;;N;;;;; 0A03;GURMUKHI SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -2530,6 +2543,12 @@ 0AF0;GUJARATI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 0AF1;GUJARATI RUPEE SIGN;Sc;0;ET;;;;;N;;;;; 0AF9;GUJARATI LETTER ZHA;Lo;0;L;;;;;N;;;;; +0AFA;GUJARATI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;; +0AFB;GUJARATI SIGN SHADDA;Mn;0;NSM;;;;;N;;;;; +0AFC;GUJARATI SIGN MADDAH;Mn;0;NSM;;;;;N;;;;; +0AFD;GUJARATI SIGN THREE-DOT NUKTA ABOVE;Mn;0;NSM;;;;;N;;;;; +0AFE;GUJARATI SIGN CIRCLE NUKTA ABOVE;Mn;0;NSM;;;;;N;;;;; +0AFF;GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE;Mn;0;NSM;;;;;N;;;;; 0B01;ORIYA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0B02;ORIYA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; 0B03;ORIYA SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -2876,6 +2895,7 @@ 0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; 0D03;MALAYALAM SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -2931,6 +2951,8 @@ 0D38;MALAYALAM LETTER SA;Lo;0;L;;;;;N;;;;; 0D39;MALAYALAM LETTER HA;Lo;0;L;;;;;N;;;;; 0D3A;MALAYALAM LETTER TTTA;Lo;0;L;;;;;N;;;;; +0D3B;MALAYALAM SIGN VERTICAL BAR VIRAMA;Mn;9;NSM;;;;;N;;;;; +0D3C;MALAYALAM SIGN CIRCULAR VIRAMA;Mn;9;NSM;;;;;N;;;;; 0D3D;MALAYALAM SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; 0D3E;MALAYALAM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; 0D3F;MALAYALAM VOWEL SIGN I;Mc;0;L;;;;;N;;;;; @@ -6413,6 +6435,7 @@ 1CF4;VEDIC TONE CANDRA ABOVE;Mn;230;NSM;;;;;N;;;;; 1CF5;VEDIC SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 1CF6;VEDIC SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +1CF7;VEDIC SIGN ATIKRAMA;Mc;0;L;;;;;N;;;;; 1CF8;VEDIC TONE RING ABOVE;Mn;230;NSM;;;;;N;;;;; 1CF9;VEDIC TONE DOUBLE RING ABOVE;Mn;230;NSM;;;;;N;;;;; 1D00;LATIN LETTER SMALL CAPITAL A;Ll;0;L;;;;;N;;;;; @@ -6661,6 +6684,10 @@ 1DF3;COMBINING LATIN SMALL LETTER O WITH DIAERESIS;Mn;230;NSM;;;;;N;;;;; 1DF4;COMBINING LATIN SMALL LETTER U WITH DIAERESIS;Mn;230;NSM;;;;;N;;;;; 1DF5;COMBINING UP TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1DF6;COMBINING KAVYKA ABOVE RIGHT;Mn;232;NSM;;;;;N;;;;; +1DF7;COMBINING KAVYKA ABOVE LEFT;Mn;228;NSM;;;;;N;;;;; +1DF8;COMBINING DOT ABOVE LEFT;Mn;228;NSM;;;;;N;;;;; +1DF9;COMBINING WIDE INVERTED BRIDGE BELOW;Mn;220;NSM;;;;;N;;;;; 1DFB;COMBINING DELETION MARK;Mn;230;NSM;;;;;N;;;;; 1DFC;COMBINING DOUBLE INVERTED BREVE BELOW;Mn;233;NSM;;;;;N;;;;; 1DFD;COMBINING ALMOST EQUAL TO BELOW;Mn;220;NSM;;;;;N;;;;; @@ -7339,6 +7366,7 @@ 20BC;MANAT SIGN;Sc;0;ET;;;;;N;;;;; 20BD;RUBLE SIGN;Sc;0;ET;;;;;N;;;;; 20BE;LARI SIGN;Sc;0;ET;;;;;N;;;;; +20BF;BITCOIN SIGN;Sc;0;ET;;;;;N;;;;; 20D0;COMBINING LEFT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT HARPOON ABOVE;;;; 20D1;COMBINING RIGHT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RIGHT HARPOON ABOVE;;;; 20D2;COMBINING LONG VERTICAL LINE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG VERTICAL BAR OVERLAY;;;; @@ -8135,6 +8163,7 @@ 23FC;POWER ON-OFF SYMBOL;So;0;ON;;;;;N;;;;; 23FD;POWER ON SYMBOL;So;0;ON;;;;;N;;;;; 23FE;POWER SLEEP SYMBOL;So;0;ON;;;;;N;;;;; +23FF;OBSERVER EYE SYMBOL;So;0;ON;;;;;N;;;;; 2400;SYMBOL FOR NULL;So;0;ON;;;;;N;GRAPHIC FOR NULL;;;; 2401;SYMBOL FOR START OF HEADING;So;0;ON;;;;;N;GRAPHIC FOR START OF HEADING;;;; 2402;SYMBOL FOR START OF TEXT;So;0;ON;;;;;N;GRAPHIC FOR START OF TEXT;;;; @@ -10083,6 +10112,7 @@ 2BCF;ROTATED WHITE FOUR POINTED CUSP;So;0;ON;;;;;N;;;;; 2BD0;SQUARE POSITION INDICATOR;So;0;ON;;;;;N;;;;; 2BD1;UNCERTAINTY SIGN;So;0;ON;;;;;N;;;;; +2BD2;GROUP MARK;So;0;ON;;;;;N;;;;; 2BEC;LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; 2BED;UPWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; 2BEE;RIGHTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;; @@ -10615,6 +10645,11 @@ 2E42;DOUBLE LOW-REVERSED-9 QUOTATION MARK;Ps;0;ON;;;;;N;;;;; 2E43;DASH WITH LEFT UPTURN;Po;0;ON;;;;;N;;;;; 2E44;DOUBLE SUSPENSION MARK;Po;0;ON;;;;;N;;;;; +2E45;INVERTED LOW KAVYKA;Po;0;ON;;;;;N;;;;; +2E46;INVERTED LOW KAVYKA WITH KAVYKA ABOVE;Po;0;ON;;;;;N;;;;; +2E47;LOW KAVYKA;Po;0;ON;;;;;N;;;;; +2E48;LOW KAVYKA WITH DOT;Po;0;ON;;;;;N;;;;; +2E49;DOUBLE STACKED COMMA;Po;0;ON;;;;;N;;;;; 2E80;CJK RADICAL REPEAT;So;0;ON;;;;;N;;;;; 2E81;CJK RADICAL CLIFF;So;0;ON;;;;;N;;;;; 2E82;CJK RADICAL SECOND ONE;So;0;ON;;;;;N;;;;; @@ -11250,6 +11285,7 @@ 312B;BOPOMOFO LETTER NG;Lo;0;L;;;;;N;;;;; 312C;BOPOMOFO LETTER GN;Lo;0;L;;;;;N;;;;; 312D;BOPOMOFO LETTER IH;Lo;0;L;;;;;N;;;;; +312E;BOPOMOFO LETTER O WITH DOT ABOVE;Lo;0;L;;;;;N;;;;; 3131;HANGUL LETTER KIYEOK;Lo;0;L;<compat> 1100;;;;N;HANGUL LETTER GIYEOG;;;; 3132;HANGUL LETTER SSANGKIYEOK;Lo;0;L;<compat> 1101;;;;N;HANGUL LETTER SSANG GIYEOG;;;; 3133;HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; @@ -12016,7 +12052,7 @@ 4DFE;HEXAGRAM FOR AFTER COMPLETION;So;0;ON;;;;;N;;;;; 4DFF;HEXAGRAM FOR BEFORE COMPLETION;So;0;ON;;;;;N;;;;; 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;; -9FD5;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;; +9FEA;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;; A000;YI SYLLABLE IT;Lo;0;L;;;;;N;;;;; A001;YI SYLLABLE IX;Lo;0;L;;;;;N;;;;; A002;YI SYLLABLE I;Lo;0;L;;;;;N;;;;; @@ -17093,6 +17129,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10321;OLD ITALIC NUMERAL FIVE;No;0;L;;;;5;N;;;;; 10322;OLD ITALIC NUMERAL TEN;No;0;L;;;;10;N;;;;; 10323;OLD ITALIC NUMERAL FIFTY;No;0;L;;;;50;N;;;;; +1032D;OLD ITALIC LETTER YE;Lo;0;L;;;;;N;;;;; +1032E;OLD ITALIC LETTER NORTHERN TSE;Lo;0;L;;;;;N;;;;; +1032F;OLD ITALIC LETTER SOUTHERN TSE;Lo;0;L;;;;;N;;;;; 10330;GOTHIC LETTER AHSA;Lo;0;L;;;;;N;;;;; 10331;GOTHIC LETTER BAIRKAN;Lo;0;L;;;;;N;;;;; 10332;GOTHIC LETTER GIBA;Lo;0;L;;;;;N;;;;; @@ -20068,6 +20107,158 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 118F1;WARANG CITI NUMBER EIGHTY;No;0;L;;;;80;N;;;;; 118F2;WARANG CITI NUMBER NINETY;No;0;L;;;;90;N;;;;; 118FF;WARANG CITI OM;Lo;0;L;;;;;N;;;;; +11A00;ZANABAZAR SQUARE LETTER A;Lo;0;L;;;;;N;;;;; +11A01;ZANABAZAR SQUARE VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11A02;ZANABAZAR SQUARE VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +11A03;ZANABAZAR SQUARE VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11A04;ZANABAZAR SQUARE VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +11A05;ZANABAZAR SQUARE VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; +11A06;ZANABAZAR SQUARE VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +11A07;ZANABAZAR SQUARE VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11A08;ZANABAZAR SQUARE VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +11A09;ZANABAZAR SQUARE VOWEL SIGN REVERSED I;Mn;0;NSM;;;;;N;;;;; +11A0A;ZANABAZAR SQUARE VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;; +11A0B;ZANABAZAR SQUARE LETTER KA;Lo;0;L;;;;;N;;;;; +11A0C;ZANABAZAR SQUARE LETTER KHA;Lo;0;L;;;;;N;;;;; +11A0D;ZANABAZAR SQUARE LETTER GA;Lo;0;L;;;;;N;;;;; +11A0E;ZANABAZAR SQUARE LETTER GHA;Lo;0;L;;;;;N;;;;; +11A0F;ZANABAZAR SQUARE LETTER NGA;Lo;0;L;;;;;N;;;;; +11A10;ZANABAZAR SQUARE LETTER CA;Lo;0;L;;;;;N;;;;; +11A11;ZANABAZAR SQUARE LETTER CHA;Lo;0;L;;;;;N;;;;; +11A12;ZANABAZAR SQUARE LETTER JA;Lo;0;L;;;;;N;;;;; +11A13;ZANABAZAR SQUARE LETTER NYA;Lo;0;L;;;;;N;;;;; +11A14;ZANABAZAR SQUARE LETTER TTA;Lo;0;L;;;;;N;;;;; +11A15;ZANABAZAR SQUARE LETTER TTHA;Lo;0;L;;;;;N;;;;; +11A16;ZANABAZAR SQUARE LETTER DDA;Lo;0;L;;;;;N;;;;; +11A17;ZANABAZAR SQUARE LETTER DDHA;Lo;0;L;;;;;N;;;;; +11A18;ZANABAZAR SQUARE LETTER NNA;Lo;0;L;;;;;N;;;;; +11A19;ZANABAZAR SQUARE LETTER TA;Lo;0;L;;;;;N;;;;; +11A1A;ZANABAZAR SQUARE LETTER THA;Lo;0;L;;;;;N;;;;; +11A1B;ZANABAZAR SQUARE LETTER DA;Lo;0;L;;;;;N;;;;; +11A1C;ZANABAZAR SQUARE LETTER DHA;Lo;0;L;;;;;N;;;;; +11A1D;ZANABAZAR SQUARE LETTER NA;Lo;0;L;;;;;N;;;;; +11A1E;ZANABAZAR SQUARE LETTER PA;Lo;0;L;;;;;N;;;;; +11A1F;ZANABAZAR SQUARE LETTER PHA;Lo;0;L;;;;;N;;;;; +11A20;ZANABAZAR SQUARE LETTER BA;Lo;0;L;;;;;N;;;;; +11A21;ZANABAZAR SQUARE LETTER BHA;Lo;0;L;;;;;N;;;;; +11A22;ZANABAZAR SQUARE LETTER MA;Lo;0;L;;;;;N;;;;; +11A23;ZANABAZAR SQUARE LETTER TSA;Lo;0;L;;;;;N;;;;; +11A24;ZANABAZAR SQUARE LETTER TSHA;Lo;0;L;;;;;N;;;;; +11A25;ZANABAZAR SQUARE LETTER DZA;Lo;0;L;;;;;N;;;;; +11A26;ZANABAZAR SQUARE LETTER DZHA;Lo;0;L;;;;;N;;;;; +11A27;ZANABAZAR SQUARE LETTER ZHA;Lo;0;L;;;;;N;;;;; +11A28;ZANABAZAR SQUARE LETTER ZA;Lo;0;L;;;;;N;;;;; +11A29;ZANABAZAR SQUARE LETTER -A;Lo;0;L;;;;;N;;;;; +11A2A;ZANABAZAR SQUARE LETTER YA;Lo;0;L;;;;;N;;;;; +11A2B;ZANABAZAR SQUARE LETTER RA;Lo;0;L;;;;;N;;;;; +11A2C;ZANABAZAR SQUARE LETTER LA;Lo;0;L;;;;;N;;;;; +11A2D;ZANABAZAR SQUARE LETTER VA;Lo;0;L;;;;;N;;;;; +11A2E;ZANABAZAR SQUARE LETTER SHA;Lo;0;L;;;;;N;;;;; +11A2F;ZANABAZAR SQUARE LETTER SSA;Lo;0;L;;;;;N;;;;; +11A30;ZANABAZAR SQUARE LETTER SA;Lo;0;L;;;;;N;;;;; +11A31;ZANABAZAR SQUARE LETTER HA;Lo;0;L;;;;;N;;;;; +11A32;ZANABAZAR SQUARE LETTER KSSA;Lo;0;L;;;;;N;;;;; +11A33;ZANABAZAR SQUARE FINAL CONSONANT MARK;Mn;0;NSM;;;;;N;;;;; +11A34;ZANABAZAR SQUARE SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +11A35;ZANABAZAR SQUARE SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11A36;ZANABAZAR SQUARE SIGN CANDRABINDU WITH ORNAMENT;Mn;0;NSM;;;;;N;;;;; +11A37;ZANABAZAR SQUARE SIGN CANDRA WITH ORNAMENT;Mn;0;NSM;;;;;N;;;;; +11A38;ZANABAZAR SQUARE SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11A39;ZANABAZAR SQUARE SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11A3A;ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA;Lo;0;L;;;;;N;;;;; +11A3B;ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA;Mn;0;NSM;;;;;N;;;;; +11A3C;ZANABAZAR SQUARE CLUSTER-FINAL LETTER RA;Mn;0;NSM;;;;;N;;;;; +11A3D;ZANABAZAR SQUARE CLUSTER-FINAL LETTER LA;Mn;0;NSM;;;;;N;;;;; +11A3E;ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA;Mn;0;NSM;;;;;N;;;;; +11A3F;ZANABAZAR SQUARE INITIAL HEAD MARK;Po;0;L;;;;;N;;;;; +11A40;ZANABAZAR SQUARE CLOSING HEAD MARK;Po;0;L;;;;;N;;;;; +11A41;ZANABAZAR SQUARE MARK TSHEG;Po;0;L;;;;;N;;;;; +11A42;ZANABAZAR SQUARE MARK SHAD;Po;0;L;;;;;N;;;;; +11A43;ZANABAZAR SQUARE MARK DOUBLE SHAD;Po;0;L;;;;;N;;;;; +11A44;ZANABAZAR SQUARE MARK LONG TSHEG;Po;0;L;;;;;N;;;;; +11A45;ZANABAZAR SQUARE INITIAL DOUBLE-LINED HEAD MARK;Po;0;L;;;;;N;;;;; +11A46;ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK;Po;0;L;;;;;N;;;;; +11A47;ZANABAZAR SQUARE SUBJOINER;Mn;9;NSM;;;;;N;;;;; +11A50;SOYOMBO LETTER A;Lo;0;L;;;;;N;;;;; +11A51;SOYOMBO VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11A52;SOYOMBO VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +11A53;SOYOMBO VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11A54;SOYOMBO VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +11A55;SOYOMBO VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +11A56;SOYOMBO VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; +11A57;SOYOMBO VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11A58;SOYOMBO VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +11A59;SOYOMBO VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11A5A;SOYOMBO VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +11A5B;SOYOMBO VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;; +11A5C;SOYOMBO LETTER KA;Lo;0;L;;;;;N;;;;; +11A5D;SOYOMBO LETTER KHA;Lo;0;L;;;;;N;;;;; +11A5E;SOYOMBO LETTER GA;Lo;0;L;;;;;N;;;;; +11A5F;SOYOMBO LETTER GHA;Lo;0;L;;;;;N;;;;; +11A60;SOYOMBO LETTER NGA;Lo;0;L;;;;;N;;;;; +11A61;SOYOMBO LETTER CA;Lo;0;L;;;;;N;;;;; +11A62;SOYOMBO LETTER CHA;Lo;0;L;;;;;N;;;;; +11A63;SOYOMBO LETTER JA;Lo;0;L;;;;;N;;;;; +11A64;SOYOMBO LETTER JHA;Lo;0;L;;;;;N;;;;; +11A65;SOYOMBO LETTER NYA;Lo;0;L;;;;;N;;;;; +11A66;SOYOMBO LETTER TTA;Lo;0;L;;;;;N;;;;; +11A67;SOYOMBO LETTER TTHA;Lo;0;L;;;;;N;;;;; +11A68;SOYOMBO LETTER DDA;Lo;0;L;;;;;N;;;;; +11A69;SOYOMBO LETTER DDHA;Lo;0;L;;;;;N;;;;; +11A6A;SOYOMBO LETTER NNA;Lo;0;L;;;;;N;;;;; +11A6B;SOYOMBO LETTER TA;Lo;0;L;;;;;N;;;;; +11A6C;SOYOMBO LETTER THA;Lo;0;L;;;;;N;;;;; +11A6D;SOYOMBO LETTER DA;Lo;0;L;;;;;N;;;;; +11A6E;SOYOMBO LETTER DHA;Lo;0;L;;;;;N;;;;; +11A6F;SOYOMBO LETTER NA;Lo;0;L;;;;;N;;;;; +11A70;SOYOMBO LETTER PA;Lo;0;L;;;;;N;;;;; +11A71;SOYOMBO LETTER PHA;Lo;0;L;;;;;N;;;;; +11A72;SOYOMBO LETTER BA;Lo;0;L;;;;;N;;;;; +11A73;SOYOMBO LETTER BHA;Lo;0;L;;;;;N;;;;; +11A74;SOYOMBO LETTER MA;Lo;0;L;;;;;N;;;;; +11A75;SOYOMBO LETTER TSA;Lo;0;L;;;;;N;;;;; +11A76;SOYOMBO LETTER TSHA;Lo;0;L;;;;;N;;;;; +11A77;SOYOMBO LETTER DZA;Lo;0;L;;;;;N;;;;; +11A78;SOYOMBO LETTER ZHA;Lo;0;L;;;;;N;;;;; +11A79;SOYOMBO LETTER ZA;Lo;0;L;;;;;N;;;;; +11A7A;SOYOMBO LETTER -A;Lo;0;L;;;;;N;;;;; +11A7B;SOYOMBO LETTER YA;Lo;0;L;;;;;N;;;;; +11A7C;SOYOMBO LETTER RA;Lo;0;L;;;;;N;;;;; +11A7D;SOYOMBO LETTER LA;Lo;0;L;;;;;N;;;;; +11A7E;SOYOMBO LETTER VA;Lo;0;L;;;;;N;;;;; +11A7F;SOYOMBO LETTER SHA;Lo;0;L;;;;;N;;;;; +11A80;SOYOMBO LETTER SSA;Lo;0;L;;;;;N;;;;; +11A81;SOYOMBO LETTER SA;Lo;0;L;;;;;N;;;;; +11A82;SOYOMBO LETTER HA;Lo;0;L;;;;;N;;;;; +11A83;SOYOMBO LETTER KSSA;Lo;0;L;;;;;N;;;;; +11A86;SOYOMBO CLUSTER-INITIAL LETTER RA;Lo;0;L;;;;;N;;;;; +11A87;SOYOMBO CLUSTER-INITIAL LETTER LA;Lo;0;L;;;;;N;;;;; +11A88;SOYOMBO CLUSTER-INITIAL LETTER SHA;Lo;0;L;;;;;N;;;;; +11A89;SOYOMBO CLUSTER-INITIAL LETTER SA;Lo;0;L;;;;;N;;;;; +11A8A;SOYOMBO FINAL CONSONANT SIGN G;Mn;0;NSM;;;;;N;;;;; +11A8B;SOYOMBO FINAL CONSONANT SIGN K;Mn;0;NSM;;;;;N;;;;; +11A8C;SOYOMBO FINAL CONSONANT SIGN NG;Mn;0;NSM;;;;;N;;;;; +11A8D;SOYOMBO FINAL CONSONANT SIGN D;Mn;0;NSM;;;;;N;;;;; +11A8E;SOYOMBO FINAL CONSONANT SIGN N;Mn;0;NSM;;;;;N;;;;; +11A8F;SOYOMBO FINAL CONSONANT SIGN B;Mn;0;NSM;;;;;N;;;;; +11A90;SOYOMBO FINAL CONSONANT SIGN M;Mn;0;NSM;;;;;N;;;;; +11A91;SOYOMBO FINAL CONSONANT SIGN R;Mn;0;NSM;;;;;N;;;;; +11A92;SOYOMBO FINAL CONSONANT SIGN L;Mn;0;NSM;;;;;N;;;;; +11A93;SOYOMBO FINAL CONSONANT SIGN SH;Mn;0;NSM;;;;;N;;;;; +11A94;SOYOMBO FINAL CONSONANT SIGN S;Mn;0;NSM;;;;;N;;;;; +11A95;SOYOMBO FINAL CONSONANT SIGN -A;Mn;0;NSM;;;;;N;;;;; +11A96;SOYOMBO SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11A97;SOYOMBO SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11A98;SOYOMBO GEMINATION MARK;Mn;0;NSM;;;;;N;;;;; +11A99;SOYOMBO SUBJOINER;Mn;9;NSM;;;;;N;;;;; +11A9A;SOYOMBO MARK TSHEG;Po;0;L;;;;;N;;;;; +11A9B;SOYOMBO MARK SHAD;Po;0;L;;;;;N;;;;; +11A9C;SOYOMBO MARK DOUBLE SHAD;Po;0;L;;;;;N;;;;; +11A9E;SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME;Po;0;L;;;;;N;;;;; +11A9F;SOYOMBO HEAD MARK WITH MOON AND SUN AND FLAME;Po;0;L;;;;;N;;;;; +11AA0;SOYOMBO HEAD MARK WITH MOON AND SUN;Po;0;L;;;;;N;;;;; +11AA1;SOYOMBO TERMINAL MARK-1;Po;0;L;;;;;N;;;;; +11AA2;SOYOMBO TERMINAL MARK-2;Po;0;L;;;;;N;;;;; 11AC0;PAU CIN HAU LETTER PA;Lo;0;L;;;;;N;;;;; 11AC1;PAU CIN HAU LETTER KA;Lo;0;L;;;;;N;;;;; 11AC2;PAU CIN HAU LETTER LA;Lo;0;L;;;;;N;;;;; @@ -20290,6 +20481,81 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11CB4;MARCHEN VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11CB5;MARCHEN SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; 11CB6;MARCHEN SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11D00;MASARAM GONDI LETTER A;Lo;0;L;;;;;N;;;;; +11D01;MASARAM GONDI LETTER AA;Lo;0;L;;;;;N;;;;; +11D02;MASARAM GONDI LETTER I;Lo;0;L;;;;;N;;;;; +11D03;MASARAM GONDI LETTER II;Lo;0;L;;;;;N;;;;; +11D04;MASARAM GONDI LETTER U;Lo;0;L;;;;;N;;;;; +11D05;MASARAM GONDI LETTER UU;Lo;0;L;;;;;N;;;;; +11D06;MASARAM GONDI LETTER E;Lo;0;L;;;;;N;;;;; +11D08;MASARAM GONDI LETTER AI;Lo;0;L;;;;;N;;;;; +11D09;MASARAM GONDI LETTER O;Lo;0;L;;;;;N;;;;; +11D0B;MASARAM GONDI LETTER AU;Lo;0;L;;;;;N;;;;; +11D0C;MASARAM GONDI LETTER KA;Lo;0;L;;;;;N;;;;; +11D0D;MASARAM GONDI LETTER KHA;Lo;0;L;;;;;N;;;;; +11D0E;MASARAM GONDI LETTER GA;Lo;0;L;;;;;N;;;;; +11D0F;MASARAM GONDI LETTER GHA;Lo;0;L;;;;;N;;;;; +11D10;MASARAM GONDI LETTER NGA;Lo;0;L;;;;;N;;;;; +11D11;MASARAM GONDI LETTER CA;Lo;0;L;;;;;N;;;;; +11D12;MASARAM GONDI LETTER CHA;Lo;0;L;;;;;N;;;;; +11D13;MASARAM GONDI LETTER JA;Lo;0;L;;;;;N;;;;; +11D14;MASARAM GONDI LETTER JHA;Lo;0;L;;;;;N;;;;; +11D15;MASARAM GONDI LETTER NYA;Lo;0;L;;;;;N;;;;; +11D16;MASARAM GONDI LETTER TTA;Lo;0;L;;;;;N;;;;; +11D17;MASARAM GONDI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11D18;MASARAM GONDI LETTER DDA;Lo;0;L;;;;;N;;;;; +11D19;MASARAM GONDI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11D1A;MASARAM GONDI LETTER NNA;Lo;0;L;;;;;N;;;;; +11D1B;MASARAM GONDI LETTER TA;Lo;0;L;;;;;N;;;;; +11D1C;MASARAM GONDI LETTER THA;Lo;0;L;;;;;N;;;;; +11D1D;MASARAM GONDI LETTER DA;Lo;0;L;;;;;N;;;;; +11D1E;MASARAM GONDI LETTER DHA;Lo;0;L;;;;;N;;;;; +11D1F;MASARAM GONDI LETTER NA;Lo;0;L;;;;;N;;;;; +11D20;MASARAM GONDI LETTER PA;Lo;0;L;;;;;N;;;;; +11D21;MASARAM GONDI LETTER PHA;Lo;0;L;;;;;N;;;;; +11D22;MASARAM GONDI LETTER BA;Lo;0;L;;;;;N;;;;; +11D23;MASARAM GONDI LETTER BHA;Lo;0;L;;;;;N;;;;; +11D24;MASARAM GONDI LETTER MA;Lo;0;L;;;;;N;;;;; +11D25;MASARAM GONDI LETTER YA;Lo;0;L;;;;;N;;;;; +11D26;MASARAM GONDI LETTER RA;Lo;0;L;;;;;N;;;;; +11D27;MASARAM GONDI LETTER LA;Lo;0;L;;;;;N;;;;; +11D28;MASARAM GONDI LETTER VA;Lo;0;L;;;;;N;;;;; +11D29;MASARAM GONDI LETTER SHA;Lo;0;L;;;;;N;;;;; +11D2A;MASARAM GONDI LETTER SSA;Lo;0;L;;;;;N;;;;; +11D2B;MASARAM GONDI LETTER SA;Lo;0;L;;;;;N;;;;; +11D2C;MASARAM GONDI LETTER HA;Lo;0;L;;;;;N;;;;; +11D2D;MASARAM GONDI LETTER LLA;Lo;0;L;;;;;N;;;;; +11D2E;MASARAM GONDI LETTER KSSA;Lo;0;L;;;;;N;;;;; +11D2F;MASARAM GONDI LETTER JNYA;Lo;0;L;;;;;N;;;;; +11D30;MASARAM GONDI LETTER TRA;Lo;0;L;;;;;N;;;;; +11D31;MASARAM GONDI VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +11D32;MASARAM GONDI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11D33;MASARAM GONDI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +11D34;MASARAM GONDI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11D35;MASARAM GONDI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11D36;MASARAM GONDI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11D3A;MASARAM GONDI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +11D3C;MASARAM GONDI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +11D3D;MASARAM GONDI VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;; +11D3F;MASARAM GONDI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; +11D40;MASARAM GONDI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11D41;MASARAM GONDI SIGN VISARGA;Mn;0;NSM;;;;;N;;;;; +11D42;MASARAM GONDI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +11D43;MASARAM GONDI SIGN CANDRA;Mn;0;NSM;;;;;N;;;;; +11D44;MASARAM GONDI SIGN HALANTA;Mn;9;NSM;;;;;N;;;;; +11D45;MASARAM GONDI VIRAMA;Mn;9;NSM;;;;;N;;;;; +11D46;MASARAM GONDI REPHA;Lo;0;L;;;;;N;;;;; +11D47;MASARAM GONDI RA-KARA;Mn;0;NSM;;;;;N;;;;; +11D50;MASARAM GONDI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11D51;MASARAM GONDI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11D52;MASARAM GONDI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11D53;MASARAM GONDI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11D54;MASARAM GONDI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11D55;MASARAM GONDI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11D56;MASARAM GONDI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11D57;MASARAM GONDI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11D58;MASARAM GONDI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11D59;MASARAM GONDI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 12000;CUNEIFORM SIGN A;Lo;0;L;;;;;N;;;;; 12001;CUNEIFORM SIGN A TIMES A;Lo;0;L;;;;;N;;;;; 12002;CUNEIFORM SIGN A TIMES BAD;Lo;0;L;;;;;N;;;;; @@ -24087,6 +24353,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16F9E;MIAO LETTER REFORMED TONE-6;Lm;0;L;;;;;N;;;;; 16F9F;MIAO LETTER REFORMED TONE-8;Lm;0;L;;;;;N;;;;; 16FE0;TANGUT ITERATION MARK;Lm;0;L;;;;;N;;;;; +16FE1;NUSHU ITERATION MARK;Lm;0;L;;;;;N;;;;; 17000;<Tangut Ideograph, First>;Lo;0;L;;;;;N;;;;; 187EC;<Tangut Ideograph, Last>;Lo;0;L;;;;;N;;;;; 18800;TANGUT COMPONENT-001;Lo;0;L;;;;;N;;;;; @@ -24846,6 +25113,687 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 18AF2;TANGUT COMPONENT-755;Lo;0;L;;;;;N;;;;; 1B000;KATAKANA LETTER ARCHAIC E;Lo;0;L;;;;;N;;;;; 1B001;HIRAGANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; +1B002;HENTAIGANA LETTER A-1;Lo;0;L;;;;;N;;;;; +1B003;HENTAIGANA LETTER A-2;Lo;0;L;;;;;N;;;;; +1B004;HENTAIGANA LETTER A-3;Lo;0;L;;;;;N;;;;; +1B005;HENTAIGANA LETTER A-WO;Lo;0;L;;;;;N;;;;; +1B006;HENTAIGANA LETTER I-1;Lo;0;L;;;;;N;;;;; +1B007;HENTAIGANA LETTER I-2;Lo;0;L;;;;;N;;;;; +1B008;HENTAIGANA LETTER I-3;Lo;0;L;;;;;N;;;;; +1B009;HENTAIGANA LETTER I-4;Lo;0;L;;;;;N;;;;; +1B00A;HENTAIGANA LETTER U-1;Lo;0;L;;;;;N;;;;; +1B00B;HENTAIGANA LETTER U-2;Lo;0;L;;;;;N;;;;; +1B00C;HENTAIGANA LETTER U-3;Lo;0;L;;;;;N;;;;; +1B00D;HENTAIGANA LETTER U-4;Lo;0;L;;;;;N;;;;; +1B00E;HENTAIGANA LETTER U-5;Lo;0;L;;;;;N;;;;; +1B00F;HENTAIGANA LETTER E-2;Lo;0;L;;;;;N;;;;; +1B010;HENTAIGANA LETTER E-3;Lo;0;L;;;;;N;;;;; +1B011;HENTAIGANA LETTER E-4;Lo;0;L;;;;;N;;;;; +1B012;HENTAIGANA LETTER E-5;Lo;0;L;;;;;N;;;;; +1B013;HENTAIGANA LETTER E-6;Lo;0;L;;;;;N;;;;; +1B014;HENTAIGANA LETTER O-1;Lo;0;L;;;;;N;;;;; +1B015;HENTAIGANA LETTER O-2;Lo;0;L;;;;;N;;;;; +1B016;HENTAIGANA LETTER O-3;Lo;0;L;;;;;N;;;;; +1B017;HENTAIGANA LETTER KA-1;Lo;0;L;;;;;N;;;;; +1B018;HENTAIGANA LETTER KA-2;Lo;0;L;;;;;N;;;;; +1B019;HENTAIGANA LETTER KA-3;Lo;0;L;;;;;N;;;;; +1B01A;HENTAIGANA LETTER KA-4;Lo;0;L;;;;;N;;;;; +1B01B;HENTAIGANA LETTER KA-5;Lo;0;L;;;;;N;;;;; +1B01C;HENTAIGANA LETTER KA-6;Lo;0;L;;;;;N;;;;; +1B01D;HENTAIGANA LETTER KA-7;Lo;0;L;;;;;N;;;;; +1B01E;HENTAIGANA LETTER KA-8;Lo;0;L;;;;;N;;;;; +1B01F;HENTAIGANA LETTER KA-9;Lo;0;L;;;;;N;;;;; +1B020;HENTAIGANA LETTER KA-10;Lo;0;L;;;;;N;;;;; +1B021;HENTAIGANA LETTER KA-11;Lo;0;L;;;;;N;;;;; +1B022;HENTAIGANA LETTER KA-KE;Lo;0;L;;;;;N;;;;; +1B023;HENTAIGANA LETTER KI-1;Lo;0;L;;;;;N;;;;; +1B024;HENTAIGANA LETTER KI-2;Lo;0;L;;;;;N;;;;; +1B025;HENTAIGANA LETTER KI-3;Lo;0;L;;;;;N;;;;; +1B026;HENTAIGANA LETTER KI-4;Lo;0;L;;;;;N;;;;; +1B027;HENTAIGANA LETTER KI-5;Lo;0;L;;;;;N;;;;; +1B028;HENTAIGANA LETTER KI-6;Lo;0;L;;;;;N;;;;; +1B029;HENTAIGANA LETTER KI-7;Lo;0;L;;;;;N;;;;; +1B02A;HENTAIGANA LETTER KI-8;Lo;0;L;;;;;N;;;;; +1B02B;HENTAIGANA LETTER KU-1;Lo;0;L;;;;;N;;;;; +1B02C;HENTAIGANA LETTER KU-2;Lo;0;L;;;;;N;;;;; +1B02D;HENTAIGANA LETTER KU-3;Lo;0;L;;;;;N;;;;; +1B02E;HENTAIGANA LETTER KU-4;Lo;0;L;;;;;N;;;;; +1B02F;HENTAIGANA LETTER KU-5;Lo;0;L;;;;;N;;;;; +1B030;HENTAIGANA LETTER KU-6;Lo;0;L;;;;;N;;;;; +1B031;HENTAIGANA LETTER KU-7;Lo;0;L;;;;;N;;;;; +1B032;HENTAIGANA LETTER KE-1;Lo;0;L;;;;;N;;;;; +1B033;HENTAIGANA LETTER KE-2;Lo;0;L;;;;;N;;;;; +1B034;HENTAIGANA LETTER KE-3;Lo;0;L;;;;;N;;;;; +1B035;HENTAIGANA LETTER KE-4;Lo;0;L;;;;;N;;;;; +1B036;HENTAIGANA LETTER KE-5;Lo;0;L;;;;;N;;;;; +1B037;HENTAIGANA LETTER KE-6;Lo;0;L;;;;;N;;;;; +1B038;HENTAIGANA LETTER KO-1;Lo;0;L;;;;;N;;;;; +1B039;HENTAIGANA LETTER KO-2;Lo;0;L;;;;;N;;;;; +1B03A;HENTAIGANA LETTER KO-3;Lo;0;L;;;;;N;;;;; +1B03B;HENTAIGANA LETTER KO-KI;Lo;0;L;;;;;N;;;;; +1B03C;HENTAIGANA LETTER SA-1;Lo;0;L;;;;;N;;;;; +1B03D;HENTAIGANA LETTER SA-2;Lo;0;L;;;;;N;;;;; +1B03E;HENTAIGANA LETTER SA-3;Lo;0;L;;;;;N;;;;; +1B03F;HENTAIGANA LETTER SA-4;Lo;0;L;;;;;N;;;;; +1B040;HENTAIGANA LETTER SA-5;Lo;0;L;;;;;N;;;;; +1B041;HENTAIGANA LETTER SA-6;Lo;0;L;;;;;N;;;;; +1B042;HENTAIGANA LETTER SA-7;Lo;0;L;;;;;N;;;;; +1B043;HENTAIGANA LETTER SA-8;Lo;0;L;;;;;N;;;;; +1B044;HENTAIGANA LETTER SI-1;Lo;0;L;;;;;N;;;;; +1B045;HENTAIGANA LETTER SI-2;Lo;0;L;;;;;N;;;;; +1B046;HENTAIGANA LETTER SI-3;Lo;0;L;;;;;N;;;;; +1B047;HENTAIGANA LETTER SI-4;Lo;0;L;;;;;N;;;;; +1B048;HENTAIGANA LETTER SI-5;Lo;0;L;;;;;N;;;;; +1B049;HENTAIGANA LETTER SI-6;Lo;0;L;;;;;N;;;;; +1B04A;HENTAIGANA LETTER SU-1;Lo;0;L;;;;;N;;;;; +1B04B;HENTAIGANA LETTER SU-2;Lo;0;L;;;;;N;;;;; +1B04C;HENTAIGANA LETTER SU-3;Lo;0;L;;;;;N;;;;; +1B04D;HENTAIGANA LETTER SU-4;Lo;0;L;;;;;N;;;;; +1B04E;HENTAIGANA LETTER SU-5;Lo;0;L;;;;;N;;;;; +1B04F;HENTAIGANA LETTER SU-6;Lo;0;L;;;;;N;;;;; +1B050;HENTAIGANA LETTER SU-7;Lo;0;L;;;;;N;;;;; +1B051;HENTAIGANA LETTER SU-8;Lo;0;L;;;;;N;;;;; +1B052;HENTAIGANA LETTER SE-1;Lo;0;L;;;;;N;;;;; +1B053;HENTAIGANA LETTER SE-2;Lo;0;L;;;;;N;;;;; +1B054;HENTAIGANA LETTER SE-3;Lo;0;L;;;;;N;;;;; +1B055;HENTAIGANA LETTER SE-4;Lo;0;L;;;;;N;;;;; +1B056;HENTAIGANA LETTER SE-5;Lo;0;L;;;;;N;;;;; +1B057;HENTAIGANA LETTER SO-1;Lo;0;L;;;;;N;;;;; +1B058;HENTAIGANA LETTER SO-2;Lo;0;L;;;;;N;;;;; +1B059;HENTAIGANA LETTER SO-3;Lo;0;L;;;;;N;;;;; +1B05A;HENTAIGANA LETTER SO-4;Lo;0;L;;;;;N;;;;; +1B05B;HENTAIGANA LETTER SO-5;Lo;0;L;;;;;N;;;;; +1B05C;HENTAIGANA LETTER SO-6;Lo;0;L;;;;;N;;;;; +1B05D;HENTAIGANA LETTER SO-7;Lo;0;L;;;;;N;;;;; +1B05E;HENTAIGANA LETTER TA-1;Lo;0;L;;;;;N;;;;; +1B05F;HENTAIGANA LETTER TA-2;Lo;0;L;;;;;N;;;;; +1B060;HENTAIGANA LETTER TA-3;Lo;0;L;;;;;N;;;;; +1B061;HENTAIGANA LETTER TA-4;Lo;0;L;;;;;N;;;;; +1B062;HENTAIGANA LETTER TI-1;Lo;0;L;;;;;N;;;;; +1B063;HENTAIGANA LETTER TI-2;Lo;0;L;;;;;N;;;;; +1B064;HENTAIGANA LETTER TI-3;Lo;0;L;;;;;N;;;;; +1B065;HENTAIGANA LETTER TI-4;Lo;0;L;;;;;N;;;;; +1B066;HENTAIGANA LETTER TI-5;Lo;0;L;;;;;N;;;;; +1B067;HENTAIGANA LETTER TI-6;Lo;0;L;;;;;N;;;;; +1B068;HENTAIGANA LETTER TI-7;Lo;0;L;;;;;N;;;;; +1B069;HENTAIGANA LETTER TU-1;Lo;0;L;;;;;N;;;;; +1B06A;HENTAIGANA LETTER TU-2;Lo;0;L;;;;;N;;;;; +1B06B;HENTAIGANA LETTER TU-3;Lo;0;L;;;;;N;;;;; +1B06C;HENTAIGANA LETTER TU-4;Lo;0;L;;;;;N;;;;; +1B06D;HENTAIGANA LETTER TU-TO;Lo;0;L;;;;;N;;;;; +1B06E;HENTAIGANA LETTER TE-1;Lo;0;L;;;;;N;;;;; +1B06F;HENTAIGANA LETTER TE-2;Lo;0;L;;;;;N;;;;; +1B070;HENTAIGANA LETTER TE-3;Lo;0;L;;;;;N;;;;; +1B071;HENTAIGANA LETTER TE-4;Lo;0;L;;;;;N;;;;; +1B072;HENTAIGANA LETTER TE-5;Lo;0;L;;;;;N;;;;; +1B073;HENTAIGANA LETTER TE-6;Lo;0;L;;;;;N;;;;; +1B074;HENTAIGANA LETTER TE-7;Lo;0;L;;;;;N;;;;; +1B075;HENTAIGANA LETTER TE-8;Lo;0;L;;;;;N;;;;; +1B076;HENTAIGANA LETTER TE-9;Lo;0;L;;;;;N;;;;; +1B077;HENTAIGANA LETTER TO-1;Lo;0;L;;;;;N;;;;; +1B078;HENTAIGANA LETTER TO-2;Lo;0;L;;;;;N;;;;; +1B079;HENTAIGANA LETTER TO-3;Lo;0;L;;;;;N;;;;; +1B07A;HENTAIGANA LETTER TO-4;Lo;0;L;;;;;N;;;;; +1B07B;HENTAIGANA LETTER TO-5;Lo;0;L;;;;;N;;;;; +1B07C;HENTAIGANA LETTER TO-6;Lo;0;L;;;;;N;;;;; +1B07D;HENTAIGANA LETTER TO-RA;Lo;0;L;;;;;N;;;;; +1B07E;HENTAIGANA LETTER NA-1;Lo;0;L;;;;;N;;;;; +1B07F;HENTAIGANA LETTER NA-2;Lo;0;L;;;;;N;;;;; +1B080;HENTAIGANA LETTER NA-3;Lo;0;L;;;;;N;;;;; +1B081;HENTAIGANA LETTER NA-4;Lo;0;L;;;;;N;;;;; +1B082;HENTAIGANA LETTER NA-5;Lo;0;L;;;;;N;;;;; +1B083;HENTAIGANA LETTER NA-6;Lo;0;L;;;;;N;;;;; +1B084;HENTAIGANA LETTER NA-7;Lo;0;L;;;;;N;;;;; +1B085;HENTAIGANA LETTER NA-8;Lo;0;L;;;;;N;;;;; +1B086;HENTAIGANA LETTER NA-9;Lo;0;L;;;;;N;;;;; +1B087;HENTAIGANA LETTER NI-1;Lo;0;L;;;;;N;;;;; +1B088;HENTAIGANA LETTER NI-2;Lo;0;L;;;;;N;;;;; +1B089;HENTAIGANA LETTER NI-3;Lo;0;L;;;;;N;;;;; +1B08A;HENTAIGANA LETTER NI-4;Lo;0;L;;;;;N;;;;; +1B08B;HENTAIGANA LETTER NI-5;Lo;0;L;;;;;N;;;;; +1B08C;HENTAIGANA LETTER NI-6;Lo;0;L;;;;;N;;;;; +1B08D;HENTAIGANA LETTER NI-7;Lo;0;L;;;;;N;;;;; +1B08E;HENTAIGANA LETTER NI-TE;Lo;0;L;;;;;N;;;;; +1B08F;HENTAIGANA LETTER NU-1;Lo;0;L;;;;;N;;;;; +1B090;HENTAIGANA LETTER NU-2;Lo;0;L;;;;;N;;;;; +1B091;HENTAIGANA LETTER NU-3;Lo;0;L;;;;;N;;;;; +1B092;HENTAIGANA LETTER NE-1;Lo;0;L;;;;;N;;;;; +1B093;HENTAIGANA LETTER NE-2;Lo;0;L;;;;;N;;;;; +1B094;HENTAIGANA LETTER NE-3;Lo;0;L;;;;;N;;;;; +1B095;HENTAIGANA LETTER NE-4;Lo;0;L;;;;;N;;;;; +1B096;HENTAIGANA LETTER NE-5;Lo;0;L;;;;;N;;;;; +1B097;HENTAIGANA LETTER NE-6;Lo;0;L;;;;;N;;;;; +1B098;HENTAIGANA LETTER NE-KO;Lo;0;L;;;;;N;;;;; +1B099;HENTAIGANA LETTER NO-1;Lo;0;L;;;;;N;;;;; +1B09A;HENTAIGANA LETTER NO-2;Lo;0;L;;;;;N;;;;; +1B09B;HENTAIGANA LETTER NO-3;Lo;0;L;;;;;N;;;;; +1B09C;HENTAIGANA LETTER NO-4;Lo;0;L;;;;;N;;;;; +1B09D;HENTAIGANA LETTER NO-5;Lo;0;L;;;;;N;;;;; +1B09E;HENTAIGANA LETTER HA-1;Lo;0;L;;;;;N;;;;; +1B09F;HENTAIGANA LETTER HA-2;Lo;0;L;;;;;N;;;;; +1B0A0;HENTAIGANA LETTER HA-3;Lo;0;L;;;;;N;;;;; +1B0A1;HENTAIGANA LETTER HA-4;Lo;0;L;;;;;N;;;;; +1B0A2;HENTAIGANA LETTER HA-5;Lo;0;L;;;;;N;;;;; +1B0A3;HENTAIGANA LETTER HA-6;Lo;0;L;;;;;N;;;;; +1B0A4;HENTAIGANA LETTER HA-7;Lo;0;L;;;;;N;;;;; +1B0A5;HENTAIGANA LETTER HA-8;Lo;0;L;;;;;N;;;;; +1B0A6;HENTAIGANA LETTER HA-9;Lo;0;L;;;;;N;;;;; +1B0A7;HENTAIGANA LETTER HA-10;Lo;0;L;;;;;N;;;;; +1B0A8;HENTAIGANA LETTER HA-11;Lo;0;L;;;;;N;;;;; +1B0A9;HENTAIGANA LETTER HI-1;Lo;0;L;;;;;N;;;;; +1B0AA;HENTAIGANA LETTER HI-2;Lo;0;L;;;;;N;;;;; +1B0AB;HENTAIGANA LETTER HI-3;Lo;0;L;;;;;N;;;;; +1B0AC;HENTAIGANA LETTER HI-4;Lo;0;L;;;;;N;;;;; +1B0AD;HENTAIGANA LETTER HI-5;Lo;0;L;;;;;N;;;;; +1B0AE;HENTAIGANA LETTER HI-6;Lo;0;L;;;;;N;;;;; +1B0AF;HENTAIGANA LETTER HI-7;Lo;0;L;;;;;N;;;;; +1B0B0;HENTAIGANA LETTER HU-1;Lo;0;L;;;;;N;;;;; +1B0B1;HENTAIGANA LETTER HU-2;Lo;0;L;;;;;N;;;;; +1B0B2;HENTAIGANA LETTER HU-3;Lo;0;L;;;;;N;;;;; +1B0B3;HENTAIGANA LETTER HE-1;Lo;0;L;;;;;N;;;;; +1B0B4;HENTAIGANA LETTER HE-2;Lo;0;L;;;;;N;;;;; +1B0B5;HENTAIGANA LETTER HE-3;Lo;0;L;;;;;N;;;;; +1B0B6;HENTAIGANA LETTER HE-4;Lo;0;L;;;;;N;;;;; +1B0B7;HENTAIGANA LETTER HE-5;Lo;0;L;;;;;N;;;;; +1B0B8;HENTAIGANA LETTER HE-6;Lo;0;L;;;;;N;;;;; +1B0B9;HENTAIGANA LETTER HE-7;Lo;0;L;;;;;N;;;;; +1B0BA;HENTAIGANA LETTER HO-1;Lo;0;L;;;;;N;;;;; +1B0BB;HENTAIGANA LETTER HO-2;Lo;0;L;;;;;N;;;;; +1B0BC;HENTAIGANA LETTER HO-3;Lo;0;L;;;;;N;;;;; +1B0BD;HENTAIGANA LETTER HO-4;Lo;0;L;;;;;N;;;;; +1B0BE;HENTAIGANA LETTER HO-5;Lo;0;L;;;;;N;;;;; +1B0BF;HENTAIGANA LETTER HO-6;Lo;0;L;;;;;N;;;;; +1B0C0;HENTAIGANA LETTER HO-7;Lo;0;L;;;;;N;;;;; +1B0C1;HENTAIGANA LETTER HO-8;Lo;0;L;;;;;N;;;;; +1B0C2;HENTAIGANA LETTER MA-1;Lo;0;L;;;;;N;;;;; +1B0C3;HENTAIGANA LETTER MA-2;Lo;0;L;;;;;N;;;;; +1B0C4;HENTAIGANA LETTER MA-3;Lo;0;L;;;;;N;;;;; +1B0C5;HENTAIGANA LETTER MA-4;Lo;0;L;;;;;N;;;;; +1B0C6;HENTAIGANA LETTER MA-5;Lo;0;L;;;;;N;;;;; +1B0C7;HENTAIGANA LETTER MA-6;Lo;0;L;;;;;N;;;;; +1B0C8;HENTAIGANA LETTER MA-7;Lo;0;L;;;;;N;;;;; +1B0C9;HENTAIGANA LETTER MI-1;Lo;0;L;;;;;N;;;;; +1B0CA;HENTAIGANA LETTER MI-2;Lo;0;L;;;;;N;;;;; +1B0CB;HENTAIGANA LETTER MI-3;Lo;0;L;;;;;N;;;;; +1B0CC;HENTAIGANA LETTER MI-4;Lo;0;L;;;;;N;;;;; +1B0CD;HENTAIGANA LETTER MI-5;Lo;0;L;;;;;N;;;;; +1B0CE;HENTAIGANA LETTER MI-6;Lo;0;L;;;;;N;;;;; +1B0CF;HENTAIGANA LETTER MI-7;Lo;0;L;;;;;N;;;;; +1B0D0;HENTAIGANA LETTER MU-1;Lo;0;L;;;;;N;;;;; +1B0D1;HENTAIGANA LETTER MU-2;Lo;0;L;;;;;N;;;;; +1B0D2;HENTAIGANA LETTER MU-3;Lo;0;L;;;;;N;;;;; +1B0D3;HENTAIGANA LETTER MU-4;Lo;0;L;;;;;N;;;;; +1B0D4;HENTAIGANA LETTER ME-1;Lo;0;L;;;;;N;;;;; +1B0D5;HENTAIGANA LETTER ME-2;Lo;0;L;;;;;N;;;;; +1B0D6;HENTAIGANA LETTER ME-MA;Lo;0;L;;;;;N;;;;; +1B0D7;HENTAIGANA LETTER MO-1;Lo;0;L;;;;;N;;;;; +1B0D8;HENTAIGANA LETTER MO-2;Lo;0;L;;;;;N;;;;; +1B0D9;HENTAIGANA LETTER MO-3;Lo;0;L;;;;;N;;;;; +1B0DA;HENTAIGANA LETTER MO-4;Lo;0;L;;;;;N;;;;; +1B0DB;HENTAIGANA LETTER MO-5;Lo;0;L;;;;;N;;;;; +1B0DC;HENTAIGANA LETTER MO-6;Lo;0;L;;;;;N;;;;; +1B0DD;HENTAIGANA LETTER YA-1;Lo;0;L;;;;;N;;;;; +1B0DE;HENTAIGANA LETTER YA-2;Lo;0;L;;;;;N;;;;; +1B0DF;HENTAIGANA LETTER YA-3;Lo;0;L;;;;;N;;;;; +1B0E0;HENTAIGANA LETTER YA-4;Lo;0;L;;;;;N;;;;; +1B0E1;HENTAIGANA LETTER YA-5;Lo;0;L;;;;;N;;;;; +1B0E2;HENTAIGANA LETTER YA-YO;Lo;0;L;;;;;N;;;;; +1B0E3;HENTAIGANA LETTER YU-1;Lo;0;L;;;;;N;;;;; +1B0E4;HENTAIGANA LETTER YU-2;Lo;0;L;;;;;N;;;;; +1B0E5;HENTAIGANA LETTER YU-3;Lo;0;L;;;;;N;;;;; +1B0E6;HENTAIGANA LETTER YU-4;Lo;0;L;;;;;N;;;;; +1B0E7;HENTAIGANA LETTER YO-1;Lo;0;L;;;;;N;;;;; +1B0E8;HENTAIGANA LETTER YO-2;Lo;0;L;;;;;N;;;;; +1B0E9;HENTAIGANA LETTER YO-3;Lo;0;L;;;;;N;;;;; +1B0EA;HENTAIGANA LETTER YO-4;Lo;0;L;;;;;N;;;;; +1B0EB;HENTAIGANA LETTER YO-5;Lo;0;L;;;;;N;;;;; +1B0EC;HENTAIGANA LETTER YO-6;Lo;0;L;;;;;N;;;;; +1B0ED;HENTAIGANA LETTER RA-1;Lo;0;L;;;;;N;;;;; +1B0EE;HENTAIGANA LETTER RA-2;Lo;0;L;;;;;N;;;;; +1B0EF;HENTAIGANA LETTER RA-3;Lo;0;L;;;;;N;;;;; +1B0F0;HENTAIGANA LETTER RA-4;Lo;0;L;;;;;N;;;;; +1B0F1;HENTAIGANA LETTER RI-1;Lo;0;L;;;;;N;;;;; +1B0F2;HENTAIGANA LETTER RI-2;Lo;0;L;;;;;N;;;;; +1B0F3;HENTAIGANA LETTER RI-3;Lo;0;L;;;;;N;;;;; +1B0F4;HENTAIGANA LETTER RI-4;Lo;0;L;;;;;N;;;;; +1B0F5;HENTAIGANA LETTER RI-5;Lo;0;L;;;;;N;;;;; +1B0F6;HENTAIGANA LETTER RI-6;Lo;0;L;;;;;N;;;;; +1B0F7;HENTAIGANA LETTER RI-7;Lo;0;L;;;;;N;;;;; +1B0F8;HENTAIGANA LETTER RU-1;Lo;0;L;;;;;N;;;;; +1B0F9;HENTAIGANA LETTER RU-2;Lo;0;L;;;;;N;;;;; +1B0FA;HENTAIGANA LETTER RU-3;Lo;0;L;;;;;N;;;;; +1B0FB;HENTAIGANA LETTER RU-4;Lo;0;L;;;;;N;;;;; +1B0FC;HENTAIGANA LETTER RU-5;Lo;0;L;;;;;N;;;;; +1B0FD;HENTAIGANA LETTER RU-6;Lo;0;L;;;;;N;;;;; +1B0FE;HENTAIGANA LETTER RE-1;Lo;0;L;;;;;N;;;;; +1B0FF;HENTAIGANA LETTER RE-2;Lo;0;L;;;;;N;;;;; +1B100;HENTAIGANA LETTER RE-3;Lo;0;L;;;;;N;;;;; +1B101;HENTAIGANA LETTER RE-4;Lo;0;L;;;;;N;;;;; +1B102;HENTAIGANA LETTER RO-1;Lo;0;L;;;;;N;;;;; +1B103;HENTAIGANA LETTER RO-2;Lo;0;L;;;;;N;;;;; +1B104;HENTAIGANA LETTER RO-3;Lo;0;L;;;;;N;;;;; +1B105;HENTAIGANA LETTER RO-4;Lo;0;L;;;;;N;;;;; +1B106;HENTAIGANA LETTER RO-5;Lo;0;L;;;;;N;;;;; +1B107;HENTAIGANA LETTER RO-6;Lo;0;L;;;;;N;;;;; +1B108;HENTAIGANA LETTER WA-1;Lo;0;L;;;;;N;;;;; +1B109;HENTAIGANA LETTER WA-2;Lo;0;L;;;;;N;;;;; +1B10A;HENTAIGANA LETTER WA-3;Lo;0;L;;;;;N;;;;; +1B10B;HENTAIGANA LETTER WA-4;Lo;0;L;;;;;N;;;;; +1B10C;HENTAIGANA LETTER WA-5;Lo;0;L;;;;;N;;;;; +1B10D;HENTAIGANA LETTER WI-1;Lo;0;L;;;;;N;;;;; +1B10E;HENTAIGANA LETTER WI-2;Lo;0;L;;;;;N;;;;; +1B10F;HENTAIGANA LETTER WI-3;Lo;0;L;;;;;N;;;;; +1B110;HENTAIGANA LETTER WI-4;Lo;0;L;;;;;N;;;;; +1B111;HENTAIGANA LETTER WI-5;Lo;0;L;;;;;N;;;;; +1B112;HENTAIGANA LETTER WE-1;Lo;0;L;;;;;N;;;;; +1B113;HENTAIGANA LETTER WE-2;Lo;0;L;;;;;N;;;;; +1B114;HENTAIGANA LETTER WE-3;Lo;0;L;;;;;N;;;;; +1B115;HENTAIGANA LETTER WE-4;Lo;0;L;;;;;N;;;;; +1B116;HENTAIGANA LETTER WO-1;Lo;0;L;;;;;N;;;;; +1B117;HENTAIGANA LETTER WO-2;Lo;0;L;;;;;N;;;;; +1B118;HENTAIGANA LETTER WO-3;Lo;0;L;;;;;N;;;;; +1B119;HENTAIGANA LETTER WO-4;Lo;0;L;;;;;N;;;;; +1B11A;HENTAIGANA LETTER WO-5;Lo;0;L;;;;;N;;;;; +1B11B;HENTAIGANA LETTER WO-6;Lo;0;L;;;;;N;;;;; +1B11C;HENTAIGANA LETTER WO-7;Lo;0;L;;;;;N;;;;; +1B11D;HENTAIGANA LETTER N-MU-MO-1;Lo;0;L;;;;;N;;;;; +1B11E;HENTAIGANA LETTER N-MU-MO-2;Lo;0;L;;;;;N;;;;; +1B170;NUSHU CHARACTER-1B170;Lo;0;L;;;;;N;;;;; +1B171;NUSHU CHARACTER-1B171;Lo;0;L;;;;;N;;;;; +1B172;NUSHU CHARACTER-1B172;Lo;0;L;;;;;N;;;;; +1B173;NUSHU CHARACTER-1B173;Lo;0;L;;;;;N;;;;; +1B174;NUSHU CHARACTER-1B174;Lo;0;L;;;;;N;;;;; +1B175;NUSHU CHARACTER-1B175;Lo;0;L;;;;;N;;;;; +1B176;NUSHU CHARACTER-1B176;Lo;0;L;;;;;N;;;;; +1B177;NUSHU CHARACTER-1B177;Lo;0;L;;;;;N;;;;; +1B178;NUSHU CHARACTER-1B178;Lo;0;L;;;;;N;;;;; +1B179;NUSHU CHARACTER-1B179;Lo;0;L;;;;;N;;;;; +1B17A;NUSHU CHARACTER-1B17A;Lo;0;L;;;;;N;;;;; +1B17B;NUSHU CHARACTER-1B17B;Lo;0;L;;;;;N;;;;; +1B17C;NUSHU CHARACTER-1B17C;Lo;0;L;;;;;N;;;;; +1B17D;NUSHU CHARACTER-1B17D;Lo;0;L;;;;;N;;;;; +1B17E;NUSHU CHARACTER-1B17E;Lo;0;L;;;;;N;;;;; +1B17F;NUSHU CHARACTER-1B17F;Lo;0;L;;;;;N;;;;; +1B180;NUSHU CHARACTER-1B180;Lo;0;L;;;;;N;;;;; +1B181;NUSHU CHARACTER-1B181;Lo;0;L;;;;;N;;;;; +1B182;NUSHU CHARACTER-1B182;Lo;0;L;;;;;N;;;;; +1B183;NUSHU CHARACTER-1B183;Lo;0;L;;;;;N;;;;; +1B184;NUSHU CHARACTER-1B184;Lo;0;L;;;;;N;;;;; +1B185;NUSHU CHARACTER-1B185;Lo;0;L;;;;;N;;;;; +1B186;NUSHU CHARACTER-1B186;Lo;0;L;;;;;N;;;;; +1B187;NUSHU CHARACTER-1B187;Lo;0;L;;;;;N;;;;; +1B188;NUSHU CHARACTER-1B188;Lo;0;L;;;;;N;;;;; +1B189;NUSHU CHARACTER-1B189;Lo;0;L;;;;;N;;;;; +1B18A;NUSHU CHARACTER-1B18A;Lo;0;L;;;;;N;;;;; +1B18B;NUSHU CHARACTER-1B18B;Lo;0;L;;;;;N;;;;; +1B18C;NUSHU CHARACTER-1B18C;Lo;0;L;;;;;N;;;;; +1B18D;NUSHU CHARACTER-1B18D;Lo;0;L;;;;;N;;;;; +1B18E;NUSHU CHARACTER-1B18E;Lo;0;L;;;;;N;;;;; +1B18F;NUSHU CHARACTER-1B18F;Lo;0;L;;;;;N;;;;; +1B190;NUSHU CHARACTER-1B190;Lo;0;L;;;;;N;;;;; +1B191;NUSHU CHARACTER-1B191;Lo;0;L;;;;;N;;;;; +1B192;NUSHU CHARACTER-1B192;Lo;0;L;;;;;N;;;;; +1B193;NUSHU CHARACTER-1B193;Lo;0;L;;;;;N;;;;; +1B194;NUSHU CHARACTER-1B194;Lo;0;L;;;;;N;;;;; +1B195;NUSHU CHARACTER-1B195;Lo;0;L;;;;;N;;;;; +1B196;NUSHU CHARACTER-1B196;Lo;0;L;;;;;N;;;;; +1B197;NUSHU CHARACTER-1B197;Lo;0;L;;;;;N;;;;; +1B198;NUSHU CHARACTER-1B198;Lo;0;L;;;;;N;;;;; +1B199;NUSHU CHARACTER-1B199;Lo;0;L;;;;;N;;;;; +1B19A;NUSHU CHARACTER-1B19A;Lo;0;L;;;;;N;;;;; +1B19B;NUSHU CHARACTER-1B19B;Lo;0;L;;;;;N;;;;; +1B19C;NUSHU CHARACTER-1B19C;Lo;0;L;;;;;N;;;;; +1B19D;NUSHU CHARACTER-1B19D;Lo;0;L;;;;;N;;;;; +1B19E;NUSHU CHARACTER-1B19E;Lo;0;L;;;;;N;;;;; +1B19F;NUSHU CHARACTER-1B19F;Lo;0;L;;;;;N;;;;; +1B1A0;NUSHU CHARACTER-1B1A0;Lo;0;L;;;;;N;;;;; +1B1A1;NUSHU CHARACTER-1B1A1;Lo;0;L;;;;;N;;;;; +1B1A2;NUSHU CHARACTER-1B1A2;Lo;0;L;;;;;N;;;;; +1B1A3;NUSHU CHARACTER-1B1A3;Lo;0;L;;;;;N;;;;; +1B1A4;NUSHU CHARACTER-1B1A4;Lo;0;L;;;;;N;;;;; +1B1A5;NUSHU CHARACTER-1B1A5;Lo;0;L;;;;;N;;;;; +1B1A6;NUSHU CHARACTER-1B1A6;Lo;0;L;;;;;N;;;;; +1B1A7;NUSHU CHARACTER-1B1A7;Lo;0;L;;;;;N;;;;; +1B1A8;NUSHU CHARACTER-1B1A8;Lo;0;L;;;;;N;;;;; +1B1A9;NUSHU CHARACTER-1B1A9;Lo;0;L;;;;;N;;;;; +1B1AA;NUSHU CHARACTER-1B1AA;Lo;0;L;;;;;N;;;;; +1B1AB;NUSHU CHARACTER-1B1AB;Lo;0;L;;;;;N;;;;; +1B1AC;NUSHU CHARACTER-1B1AC;Lo;0;L;;;;;N;;;;; +1B1AD;NUSHU CHARACTER-1B1AD;Lo;0;L;;;;;N;;;;; +1B1AE;NUSHU CHARACTER-1B1AE;Lo;0;L;;;;;N;;;;; +1B1AF;NUSHU CHARACTER-1B1AF;Lo;0;L;;;;;N;;;;; +1B1B0;NUSHU CHARACTER-1B1B0;Lo;0;L;;;;;N;;;;; +1B1B1;NUSHU CHARACTER-1B1B1;Lo;0;L;;;;;N;;;;; +1B1B2;NUSHU CHARACTER-1B1B2;Lo;0;L;;;;;N;;;;; +1B1B3;NUSHU CHARACTER-1B1B3;Lo;0;L;;;;;N;;;;; +1B1B4;NUSHU CHARACTER-1B1B4;Lo;0;L;;;;;N;;;;; +1B1B5;NUSHU CHARACTER-1B1B5;Lo;0;L;;;;;N;;;;; +1B1B6;NUSHU CHARACTER-1B1B6;Lo;0;L;;;;;N;;;;; +1B1B7;NUSHU CHARACTER-1B1B7;Lo;0;L;;;;;N;;;;; +1B1B8;NUSHU CHARACTER-1B1B8;Lo;0;L;;;;;N;;;;; +1B1B9;NUSHU CHARACTER-1B1B9;Lo;0;L;;;;;N;;;;; +1B1BA;NUSHU CHARACTER-1B1BA;Lo;0;L;;;;;N;;;;; +1B1BB;NUSHU CHARACTER-1B1BB;Lo;0;L;;;;;N;;;;; +1B1BC;NUSHU CHARACTER-1B1BC;Lo;0;L;;;;;N;;;;; +1B1BD;NUSHU CHARACTER-1B1BD;Lo;0;L;;;;;N;;;;; +1B1BE;NUSHU CHARACTER-1B1BE;Lo;0;L;;;;;N;;;;; +1B1BF;NUSHU CHARACTER-1B1BF;Lo;0;L;;;;;N;;;;; +1B1C0;NUSHU CHARACTER-1B1C0;Lo;0;L;;;;;N;;;;; +1B1C1;NUSHU CHARACTER-1B1C1;Lo;0;L;;;;;N;;;;; +1B1C2;NUSHU CHARACTER-1B1C2;Lo;0;L;;;;;N;;;;; +1B1C3;NUSHU CHARACTER-1B1C3;Lo;0;L;;;;;N;;;;; +1B1C4;NUSHU CHARACTER-1B1C4;Lo;0;L;;;;;N;;;;; +1B1C5;NUSHU CHARACTER-1B1C5;Lo;0;L;;;;;N;;;;; +1B1C6;NUSHU CHARACTER-1B1C6;Lo;0;L;;;;;N;;;;; +1B1C7;NUSHU CHARACTER-1B1C7;Lo;0;L;;;;;N;;;;; +1B1C8;NUSHU CHARACTER-1B1C8;Lo;0;L;;;;;N;;;;; +1B1C9;NUSHU CHARACTER-1B1C9;Lo;0;L;;;;;N;;;;; +1B1CA;NUSHU CHARACTER-1B1CA;Lo;0;L;;;;;N;;;;; +1B1CB;NUSHU CHARACTER-1B1CB;Lo;0;L;;;;;N;;;;; +1B1CC;NUSHU CHARACTER-1B1CC;Lo;0;L;;;;;N;;;;; +1B1CD;NUSHU CHARACTER-1B1CD;Lo;0;L;;;;;N;;;;; +1B1CE;NUSHU CHARACTER-1B1CE;Lo;0;L;;;;;N;;;;; +1B1CF;NUSHU CHARACTER-1B1CF;Lo;0;L;;;;;N;;;;; +1B1D0;NUSHU CHARACTER-1B1D0;Lo;0;L;;;;;N;;;;; +1B1D1;NUSHU CHARACTER-1B1D1;Lo;0;L;;;;;N;;;;; +1B1D2;NUSHU CHARACTER-1B1D2;Lo;0;L;;;;;N;;;;; +1B1D3;NUSHU CHARACTER-1B1D3;Lo;0;L;;;;;N;;;;; +1B1D4;NUSHU CHARACTER-1B1D4;Lo;0;L;;;;;N;;;;; +1B1D5;NUSHU CHARACTER-1B1D5;Lo;0;L;;;;;N;;;;; +1B1D6;NUSHU CHARACTER-1B1D6;Lo;0;L;;;;;N;;;;; +1B1D7;NUSHU CHARACTER-1B1D7;Lo;0;L;;;;;N;;;;; +1B1D8;NUSHU CHARACTER-1B1D8;Lo;0;L;;;;;N;;;;; +1B1D9;NUSHU CHARACTER-1B1D9;Lo;0;L;;;;;N;;;;; +1B1DA;NUSHU CHARACTER-1B1DA;Lo;0;L;;;;;N;;;;; +1B1DB;NUSHU CHARACTER-1B1DB;Lo;0;L;;;;;N;;;;; +1B1DC;NUSHU CHARACTER-1B1DC;Lo;0;L;;;;;N;;;;; +1B1DD;NUSHU CHARACTER-1B1DD;Lo;0;L;;;;;N;;;;; +1B1DE;NUSHU CHARACTER-1B1DE;Lo;0;L;;;;;N;;;;; +1B1DF;NUSHU CHARACTER-1B1DF;Lo;0;L;;;;;N;;;;; +1B1E0;NUSHU CHARACTER-1B1E0;Lo;0;L;;;;;N;;;;; +1B1E1;NUSHU CHARACTER-1B1E1;Lo;0;L;;;;;N;;;;; +1B1E2;NUSHU CHARACTER-1B1E2;Lo;0;L;;;;;N;;;;; +1B1E3;NUSHU CHARACTER-1B1E3;Lo;0;L;;;;;N;;;;; +1B1E4;NUSHU CHARACTER-1B1E4;Lo;0;L;;;;;N;;;;; +1B1E5;NUSHU CHARACTER-1B1E5;Lo;0;L;;;;;N;;;;; +1B1E6;NUSHU CHARACTER-1B1E6;Lo;0;L;;;;;N;;;;; +1B1E7;NUSHU CHARACTER-1B1E7;Lo;0;L;;;;;N;;;;; +1B1E8;NUSHU CHARACTER-1B1E8;Lo;0;L;;;;;N;;;;; +1B1E9;NUSHU CHARACTER-1B1E9;Lo;0;L;;;;;N;;;;; +1B1EA;NUSHU CHARACTER-1B1EA;Lo;0;L;;;;;N;;;;; +1B1EB;NUSHU CHARACTER-1B1EB;Lo;0;L;;;;;N;;;;; +1B1EC;NUSHU CHARACTER-1B1EC;Lo;0;L;;;;;N;;;;; +1B1ED;NUSHU CHARACTER-1B1ED;Lo;0;L;;;;;N;;;;; +1B1EE;NUSHU CHARACTER-1B1EE;Lo;0;L;;;;;N;;;;; +1B1EF;NUSHU CHARACTER-1B1EF;Lo;0;L;;;;;N;;;;; +1B1F0;NUSHU CHARACTER-1B1F0;Lo;0;L;;;;;N;;;;; +1B1F1;NUSHU CHARACTER-1B1F1;Lo;0;L;;;;;N;;;;; +1B1F2;NUSHU CHARACTER-1B1F2;Lo;0;L;;;;;N;;;;; +1B1F3;NUSHU CHARACTER-1B1F3;Lo;0;L;;;;;N;;;;; +1B1F4;NUSHU CHARACTER-1B1F4;Lo;0;L;;;;;N;;;;; +1B1F5;NUSHU CHARACTER-1B1F5;Lo;0;L;;;;;N;;;;; +1B1F6;NUSHU CHARACTER-1B1F6;Lo;0;L;;;;;N;;;;; +1B1F7;NUSHU CHARACTER-1B1F7;Lo;0;L;;;;;N;;;;; +1B1F8;NUSHU CHARACTER-1B1F8;Lo;0;L;;;;;N;;;;; +1B1F9;NUSHU CHARACTER-1B1F9;Lo;0;L;;;;;N;;;;; +1B1FA;NUSHU CHARACTER-1B1FA;Lo;0;L;;;;;N;;;;; +1B1FB;NUSHU CHARACTER-1B1FB;Lo;0;L;;;;;N;;;;; +1B1FC;NUSHU CHARACTER-1B1FC;Lo;0;L;;;;;N;;;;; +1B1FD;NUSHU CHARACTER-1B1FD;Lo;0;L;;;;;N;;;;; +1B1FE;NUSHU CHARACTER-1B1FE;Lo;0;L;;;;;N;;;;; +1B1FF;NUSHU CHARACTER-1B1FF;Lo;0;L;;;;;N;;;;; +1B200;NUSHU CHARACTER-1B200;Lo;0;L;;;;;N;;;;; +1B201;NUSHU CHARACTER-1B201;Lo;0;L;;;;;N;;;;; +1B202;NUSHU CHARACTER-1B202;Lo;0;L;;;;;N;;;;; +1B203;NUSHU CHARACTER-1B203;Lo;0;L;;;;;N;;;;; +1B204;NUSHU CHARACTER-1B204;Lo;0;L;;;;;N;;;;; +1B205;NUSHU CHARACTER-1B205;Lo;0;L;;;;;N;;;;; +1B206;NUSHU CHARACTER-1B206;Lo;0;L;;;;;N;;;;; +1B207;NUSHU CHARACTER-1B207;Lo;0;L;;;;;N;;;;; +1B208;NUSHU CHARACTER-1B208;Lo;0;L;;;;;N;;;;; +1B209;NUSHU CHARACTER-1B209;Lo;0;L;;;;;N;;;;; +1B20A;NUSHU CHARACTER-1B20A;Lo;0;L;;;;;N;;;;; +1B20B;NUSHU CHARACTER-1B20B;Lo;0;L;;;;;N;;;;; +1B20C;NUSHU CHARACTER-1B20C;Lo;0;L;;;;;N;;;;; +1B20D;NUSHU CHARACTER-1B20D;Lo;0;L;;;;;N;;;;; +1B20E;NUSHU CHARACTER-1B20E;Lo;0;L;;;;;N;;;;; +1B20F;NUSHU CHARACTER-1B20F;Lo;0;L;;;;;N;;;;; +1B210;NUSHU CHARACTER-1B210;Lo;0;L;;;;;N;;;;; +1B211;NUSHU CHARACTER-1B211;Lo;0;L;;;;;N;;;;; +1B212;NUSHU CHARACTER-1B212;Lo;0;L;;;;;N;;;;; +1B213;NUSHU CHARACTER-1B213;Lo;0;L;;;;;N;;;;; +1B214;NUSHU CHARACTER-1B214;Lo;0;L;;;;;N;;;;; +1B215;NUSHU CHARACTER-1B215;Lo;0;L;;;;;N;;;;; +1B216;NUSHU CHARACTER-1B216;Lo;0;L;;;;;N;;;;; +1B217;NUSHU CHARACTER-1B217;Lo;0;L;;;;;N;;;;; +1B218;NUSHU CHARACTER-1B218;Lo;0;L;;;;;N;;;;; +1B219;NUSHU CHARACTER-1B219;Lo;0;L;;;;;N;;;;; +1B21A;NUSHU CHARACTER-1B21A;Lo;0;L;;;;;N;;;;; +1B21B;NUSHU CHARACTER-1B21B;Lo;0;L;;;;;N;;;;; +1B21C;NUSHU CHARACTER-1B21C;Lo;0;L;;;;;N;;;;; +1B21D;NUSHU CHARACTER-1B21D;Lo;0;L;;;;;N;;;;; +1B21E;NUSHU CHARACTER-1B21E;Lo;0;L;;;;;N;;;;; +1B21F;NUSHU CHARACTER-1B21F;Lo;0;L;;;;;N;;;;; +1B220;NUSHU CHARACTER-1B220;Lo;0;L;;;;;N;;;;; +1B221;NUSHU CHARACTER-1B221;Lo;0;L;;;;;N;;;;; +1B222;NUSHU CHARACTER-1B222;Lo;0;L;;;;;N;;;;; +1B223;NUSHU CHARACTER-1B223;Lo;0;L;;;;;N;;;;; +1B224;NUSHU CHARACTER-1B224;Lo;0;L;;;;;N;;;;; +1B225;NUSHU CHARACTER-1B225;Lo;0;L;;;;;N;;;;; +1B226;NUSHU CHARACTER-1B226;Lo;0;L;;;;;N;;;;; +1B227;NUSHU CHARACTER-1B227;Lo;0;L;;;;;N;;;;; +1B228;NUSHU CHARACTER-1B228;Lo;0;L;;;;;N;;;;; +1B229;NUSHU CHARACTER-1B229;Lo;0;L;;;;;N;;;;; +1B22A;NUSHU CHARACTER-1B22A;Lo;0;L;;;;;N;;;;; +1B22B;NUSHU CHARACTER-1B22B;Lo;0;L;;;;;N;;;;; +1B22C;NUSHU CHARACTER-1B22C;Lo;0;L;;;;;N;;;;; +1B22D;NUSHU CHARACTER-1B22D;Lo;0;L;;;;;N;;;;; +1B22E;NUSHU CHARACTER-1B22E;Lo;0;L;;;;;N;;;;; +1B22F;NUSHU CHARACTER-1B22F;Lo;0;L;;;;;N;;;;; +1B230;NUSHU CHARACTER-1B230;Lo;0;L;;;;;N;;;;; +1B231;NUSHU CHARACTER-1B231;Lo;0;L;;;;;N;;;;; +1B232;NUSHU CHARACTER-1B232;Lo;0;L;;;;;N;;;;; +1B233;NUSHU CHARACTER-1B233;Lo;0;L;;;;;N;;;;; +1B234;NUSHU CHARACTER-1B234;Lo;0;L;;;;;N;;;;; +1B235;NUSHU CHARACTER-1B235;Lo;0;L;;;;;N;;;;; +1B236;NUSHU CHARACTER-1B236;Lo;0;L;;;;;N;;;;; +1B237;NUSHU CHARACTER-1B237;Lo;0;L;;;;;N;;;;; +1B238;NUSHU CHARACTER-1B238;Lo;0;L;;;;;N;;;;; +1B239;NUSHU CHARACTER-1B239;Lo;0;L;;;;;N;;;;; +1B23A;NUSHU CHARACTER-1B23A;Lo;0;L;;;;;N;;;;; +1B23B;NUSHU CHARACTER-1B23B;Lo;0;L;;;;;N;;;;; +1B23C;NUSHU CHARACTER-1B23C;Lo;0;L;;;;;N;;;;; +1B23D;NUSHU CHARACTER-1B23D;Lo;0;L;;;;;N;;;;; +1B23E;NUSHU CHARACTER-1B23E;Lo;0;L;;;;;N;;;;; +1B23F;NUSHU CHARACTER-1B23F;Lo;0;L;;;;;N;;;;; +1B240;NUSHU CHARACTER-1B240;Lo;0;L;;;;;N;;;;; +1B241;NUSHU CHARACTER-1B241;Lo;0;L;;;;;N;;;;; +1B242;NUSHU CHARACTER-1B242;Lo;0;L;;;;;N;;;;; +1B243;NUSHU CHARACTER-1B243;Lo;0;L;;;;;N;;;;; +1B244;NUSHU CHARACTER-1B244;Lo;0;L;;;;;N;;;;; +1B245;NUSHU CHARACTER-1B245;Lo;0;L;;;;;N;;;;; +1B246;NUSHU CHARACTER-1B246;Lo;0;L;;;;;N;;;;; +1B247;NUSHU CHARACTER-1B247;Lo;0;L;;;;;N;;;;; +1B248;NUSHU CHARACTER-1B248;Lo;0;L;;;;;N;;;;; +1B249;NUSHU CHARACTER-1B249;Lo;0;L;;;;;N;;;;; +1B24A;NUSHU CHARACTER-1B24A;Lo;0;L;;;;;N;;;;; +1B24B;NUSHU CHARACTER-1B24B;Lo;0;L;;;;;N;;;;; +1B24C;NUSHU CHARACTER-1B24C;Lo;0;L;;;;;N;;;;; +1B24D;NUSHU CHARACTER-1B24D;Lo;0;L;;;;;N;;;;; +1B24E;NUSHU CHARACTER-1B24E;Lo;0;L;;;;;N;;;;; +1B24F;NUSHU CHARACTER-1B24F;Lo;0;L;;;;;N;;;;; +1B250;NUSHU CHARACTER-1B250;Lo;0;L;;;;;N;;;;; +1B251;NUSHU CHARACTER-1B251;Lo;0;L;;;;;N;;;;; +1B252;NUSHU CHARACTER-1B252;Lo;0;L;;;;;N;;;;; +1B253;NUSHU CHARACTER-1B253;Lo;0;L;;;;;N;;;;; +1B254;NUSHU CHARACTER-1B254;Lo;0;L;;;;;N;;;;; +1B255;NUSHU CHARACTER-1B255;Lo;0;L;;;;;N;;;;; +1B256;NUSHU CHARACTER-1B256;Lo;0;L;;;;;N;;;;; +1B257;NUSHU CHARACTER-1B257;Lo;0;L;;;;;N;;;;; +1B258;NUSHU CHARACTER-1B258;Lo;0;L;;;;;N;;;;; +1B259;NUSHU CHARACTER-1B259;Lo;0;L;;;;;N;;;;; +1B25A;NUSHU CHARACTER-1B25A;Lo;0;L;;;;;N;;;;; +1B25B;NUSHU CHARACTER-1B25B;Lo;0;L;;;;;N;;;;; +1B25C;NUSHU CHARACTER-1B25C;Lo;0;L;;;;;N;;;;; +1B25D;NUSHU CHARACTER-1B25D;Lo;0;L;;;;;N;;;;; +1B25E;NUSHU CHARACTER-1B25E;Lo;0;L;;;;;N;;;;; +1B25F;NUSHU CHARACTER-1B25F;Lo;0;L;;;;;N;;;;; +1B260;NUSHU CHARACTER-1B260;Lo;0;L;;;;;N;;;;; +1B261;NUSHU CHARACTER-1B261;Lo;0;L;;;;;N;;;;; +1B262;NUSHU CHARACTER-1B262;Lo;0;L;;;;;N;;;;; +1B263;NUSHU CHARACTER-1B263;Lo;0;L;;;;;N;;;;; +1B264;NUSHU CHARACTER-1B264;Lo;0;L;;;;;N;;;;; +1B265;NUSHU CHARACTER-1B265;Lo;0;L;;;;;N;;;;; +1B266;NUSHU CHARACTER-1B266;Lo;0;L;;;;;N;;;;; +1B267;NUSHU CHARACTER-1B267;Lo;0;L;;;;;N;;;;; +1B268;NUSHU CHARACTER-1B268;Lo;0;L;;;;;N;;;;; +1B269;NUSHU CHARACTER-1B269;Lo;0;L;;;;;N;;;;; +1B26A;NUSHU CHARACTER-1B26A;Lo;0;L;;;;;N;;;;; +1B26B;NUSHU CHARACTER-1B26B;Lo;0;L;;;;;N;;;;; +1B26C;NUSHU CHARACTER-1B26C;Lo;0;L;;;;;N;;;;; +1B26D;NUSHU CHARACTER-1B26D;Lo;0;L;;;;;N;;;;; +1B26E;NUSHU CHARACTER-1B26E;Lo;0;L;;;;;N;;;;; +1B26F;NUSHU CHARACTER-1B26F;Lo;0;L;;;;;N;;;;; +1B270;NUSHU CHARACTER-1B270;Lo;0;L;;;;;N;;;;; +1B271;NUSHU CHARACTER-1B271;Lo;0;L;;;;;N;;;;; +1B272;NUSHU CHARACTER-1B272;Lo;0;L;;;;;N;;;;; +1B273;NUSHU CHARACTER-1B273;Lo;0;L;;;;;N;;;;; +1B274;NUSHU CHARACTER-1B274;Lo;0;L;;;;;N;;;;; +1B275;NUSHU CHARACTER-1B275;Lo;0;L;;;;;N;;;;; +1B276;NUSHU CHARACTER-1B276;Lo;0;L;;;;;N;;;;; +1B277;NUSHU CHARACTER-1B277;Lo;0;L;;;;;N;;;;; +1B278;NUSHU CHARACTER-1B278;Lo;0;L;;;;;N;;;;; +1B279;NUSHU CHARACTER-1B279;Lo;0;L;;;;;N;;;;; +1B27A;NUSHU CHARACTER-1B27A;Lo;0;L;;;;;N;;;;; +1B27B;NUSHU CHARACTER-1B27B;Lo;0;L;;;;;N;;;;; +1B27C;NUSHU CHARACTER-1B27C;Lo;0;L;;;;;N;;;;; +1B27D;NUSHU CHARACTER-1B27D;Lo;0;L;;;;;N;;;;; +1B27E;NUSHU CHARACTER-1B27E;Lo;0;L;;;;;N;;;;; +1B27F;NUSHU CHARACTER-1B27F;Lo;0;L;;;;;N;;;;; +1B280;NUSHU CHARACTER-1B280;Lo;0;L;;;;;N;;;;; +1B281;NUSHU CHARACTER-1B281;Lo;0;L;;;;;N;;;;; +1B282;NUSHU CHARACTER-1B282;Lo;0;L;;;;;N;;;;; +1B283;NUSHU CHARACTER-1B283;Lo;0;L;;;;;N;;;;; +1B284;NUSHU CHARACTER-1B284;Lo;0;L;;;;;N;;;;; +1B285;NUSHU CHARACTER-1B285;Lo;0;L;;;;;N;;;;; +1B286;NUSHU CHARACTER-1B286;Lo;0;L;;;;;N;;;;; +1B287;NUSHU CHARACTER-1B287;Lo;0;L;;;;;N;;;;; +1B288;NUSHU CHARACTER-1B288;Lo;0;L;;;;;N;;;;; +1B289;NUSHU CHARACTER-1B289;Lo;0;L;;;;;N;;;;; +1B28A;NUSHU CHARACTER-1B28A;Lo;0;L;;;;;N;;;;; +1B28B;NUSHU CHARACTER-1B28B;Lo;0;L;;;;;N;;;;; +1B28C;NUSHU CHARACTER-1B28C;Lo;0;L;;;;;N;;;;; +1B28D;NUSHU CHARACTER-1B28D;Lo;0;L;;;;;N;;;;; +1B28E;NUSHU CHARACTER-1B28E;Lo;0;L;;;;;N;;;;; +1B28F;NUSHU CHARACTER-1B28F;Lo;0;L;;;;;N;;;;; +1B290;NUSHU CHARACTER-1B290;Lo;0;L;;;;;N;;;;; +1B291;NUSHU CHARACTER-1B291;Lo;0;L;;;;;N;;;;; +1B292;NUSHU CHARACTER-1B292;Lo;0;L;;;;;N;;;;; +1B293;NUSHU CHARACTER-1B293;Lo;0;L;;;;;N;;;;; +1B294;NUSHU CHARACTER-1B294;Lo;0;L;;;;;N;;;;; +1B295;NUSHU CHARACTER-1B295;Lo;0;L;;;;;N;;;;; +1B296;NUSHU CHARACTER-1B296;Lo;0;L;;;;;N;;;;; +1B297;NUSHU CHARACTER-1B297;Lo;0;L;;;;;N;;;;; +1B298;NUSHU CHARACTER-1B298;Lo;0;L;;;;;N;;;;; +1B299;NUSHU CHARACTER-1B299;Lo;0;L;;;;;N;;;;; +1B29A;NUSHU CHARACTER-1B29A;Lo;0;L;;;;;N;;;;; +1B29B;NUSHU CHARACTER-1B29B;Lo;0;L;;;;;N;;;;; +1B29C;NUSHU CHARACTER-1B29C;Lo;0;L;;;;;N;;;;; +1B29D;NUSHU CHARACTER-1B29D;Lo;0;L;;;;;N;;;;; +1B29E;NUSHU CHARACTER-1B29E;Lo;0;L;;;;;N;;;;; +1B29F;NUSHU CHARACTER-1B29F;Lo;0;L;;;;;N;;;;; +1B2A0;NUSHU CHARACTER-1B2A0;Lo;0;L;;;;;N;;;;; +1B2A1;NUSHU CHARACTER-1B2A1;Lo;0;L;;;;;N;;;;; +1B2A2;NUSHU CHARACTER-1B2A2;Lo;0;L;;;;;N;;;;; +1B2A3;NUSHU CHARACTER-1B2A3;Lo;0;L;;;;;N;;;;; +1B2A4;NUSHU CHARACTER-1B2A4;Lo;0;L;;;;;N;;;;; +1B2A5;NUSHU CHARACTER-1B2A5;Lo;0;L;;;;;N;;;;; +1B2A6;NUSHU CHARACTER-1B2A6;Lo;0;L;;;;;N;;;;; +1B2A7;NUSHU CHARACTER-1B2A7;Lo;0;L;;;;;N;;;;; +1B2A8;NUSHU CHARACTER-1B2A8;Lo;0;L;;;;;N;;;;; +1B2A9;NUSHU CHARACTER-1B2A9;Lo;0;L;;;;;N;;;;; +1B2AA;NUSHU CHARACTER-1B2AA;Lo;0;L;;;;;N;;;;; +1B2AB;NUSHU CHARACTER-1B2AB;Lo;0;L;;;;;N;;;;; +1B2AC;NUSHU CHARACTER-1B2AC;Lo;0;L;;;;;N;;;;; +1B2AD;NUSHU CHARACTER-1B2AD;Lo;0;L;;;;;N;;;;; +1B2AE;NUSHU CHARACTER-1B2AE;Lo;0;L;;;;;N;;;;; +1B2AF;NUSHU CHARACTER-1B2AF;Lo;0;L;;;;;N;;;;; +1B2B0;NUSHU CHARACTER-1B2B0;Lo;0;L;;;;;N;;;;; +1B2B1;NUSHU CHARACTER-1B2B1;Lo;0;L;;;;;N;;;;; +1B2B2;NUSHU CHARACTER-1B2B2;Lo;0;L;;;;;N;;;;; +1B2B3;NUSHU CHARACTER-1B2B3;Lo;0;L;;;;;N;;;;; +1B2B4;NUSHU CHARACTER-1B2B4;Lo;0;L;;;;;N;;;;; +1B2B5;NUSHU CHARACTER-1B2B5;Lo;0;L;;;;;N;;;;; +1B2B6;NUSHU CHARACTER-1B2B6;Lo;0;L;;;;;N;;;;; +1B2B7;NUSHU CHARACTER-1B2B7;Lo;0;L;;;;;N;;;;; +1B2B8;NUSHU CHARACTER-1B2B8;Lo;0;L;;;;;N;;;;; +1B2B9;NUSHU CHARACTER-1B2B9;Lo;0;L;;;;;N;;;;; +1B2BA;NUSHU CHARACTER-1B2BA;Lo;0;L;;;;;N;;;;; +1B2BB;NUSHU CHARACTER-1B2BB;Lo;0;L;;;;;N;;;;; +1B2BC;NUSHU CHARACTER-1B2BC;Lo;0;L;;;;;N;;;;; +1B2BD;NUSHU CHARACTER-1B2BD;Lo;0;L;;;;;N;;;;; +1B2BE;NUSHU CHARACTER-1B2BE;Lo;0;L;;;;;N;;;;; +1B2BF;NUSHU CHARACTER-1B2BF;Lo;0;L;;;;;N;;;;; +1B2C0;NUSHU CHARACTER-1B2C0;Lo;0;L;;;;;N;;;;; +1B2C1;NUSHU CHARACTER-1B2C1;Lo;0;L;;;;;N;;;;; +1B2C2;NUSHU CHARACTER-1B2C2;Lo;0;L;;;;;N;;;;; +1B2C3;NUSHU CHARACTER-1B2C3;Lo;0;L;;;;;N;;;;; +1B2C4;NUSHU CHARACTER-1B2C4;Lo;0;L;;;;;N;;;;; +1B2C5;NUSHU CHARACTER-1B2C5;Lo;0;L;;;;;N;;;;; +1B2C6;NUSHU CHARACTER-1B2C6;Lo;0;L;;;;;N;;;;; +1B2C7;NUSHU CHARACTER-1B2C7;Lo;0;L;;;;;N;;;;; +1B2C8;NUSHU CHARACTER-1B2C8;Lo;0;L;;;;;N;;;;; +1B2C9;NUSHU CHARACTER-1B2C9;Lo;0;L;;;;;N;;;;; +1B2CA;NUSHU CHARACTER-1B2CA;Lo;0;L;;;;;N;;;;; +1B2CB;NUSHU CHARACTER-1B2CB;Lo;0;L;;;;;N;;;;; +1B2CC;NUSHU CHARACTER-1B2CC;Lo;0;L;;;;;N;;;;; +1B2CD;NUSHU CHARACTER-1B2CD;Lo;0;L;;;;;N;;;;; +1B2CE;NUSHU CHARACTER-1B2CE;Lo;0;L;;;;;N;;;;; +1B2CF;NUSHU CHARACTER-1B2CF;Lo;0;L;;;;;N;;;;; +1B2D0;NUSHU CHARACTER-1B2D0;Lo;0;L;;;;;N;;;;; +1B2D1;NUSHU CHARACTER-1B2D1;Lo;0;L;;;;;N;;;;; +1B2D2;NUSHU CHARACTER-1B2D2;Lo;0;L;;;;;N;;;;; +1B2D3;NUSHU CHARACTER-1B2D3;Lo;0;L;;;;;N;;;;; +1B2D4;NUSHU CHARACTER-1B2D4;Lo;0;L;;;;;N;;;;; +1B2D5;NUSHU CHARACTER-1B2D5;Lo;0;L;;;;;N;;;;; +1B2D6;NUSHU CHARACTER-1B2D6;Lo;0;L;;;;;N;;;;; +1B2D7;NUSHU CHARACTER-1B2D7;Lo;0;L;;;;;N;;;;; +1B2D8;NUSHU CHARACTER-1B2D8;Lo;0;L;;;;;N;;;;; +1B2D9;NUSHU CHARACTER-1B2D9;Lo;0;L;;;;;N;;;;; +1B2DA;NUSHU CHARACTER-1B2DA;Lo;0;L;;;;;N;;;;; +1B2DB;NUSHU CHARACTER-1B2DB;Lo;0;L;;;;;N;;;;; +1B2DC;NUSHU CHARACTER-1B2DC;Lo;0;L;;;;;N;;;;; +1B2DD;NUSHU CHARACTER-1B2DD;Lo;0;L;;;;;N;;;;; +1B2DE;NUSHU CHARACTER-1B2DE;Lo;0;L;;;;;N;;;;; +1B2DF;NUSHU CHARACTER-1B2DF;Lo;0;L;;;;;N;;;;; +1B2E0;NUSHU CHARACTER-1B2E0;Lo;0;L;;;;;N;;;;; +1B2E1;NUSHU CHARACTER-1B2E1;Lo;0;L;;;;;N;;;;; +1B2E2;NUSHU CHARACTER-1B2E2;Lo;0;L;;;;;N;;;;; +1B2E3;NUSHU CHARACTER-1B2E3;Lo;0;L;;;;;N;;;;; +1B2E4;NUSHU CHARACTER-1B2E4;Lo;0;L;;;;;N;;;;; +1B2E5;NUSHU CHARACTER-1B2E5;Lo;0;L;;;;;N;;;;; +1B2E6;NUSHU CHARACTER-1B2E6;Lo;0;L;;;;;N;;;;; +1B2E7;NUSHU CHARACTER-1B2E7;Lo;0;L;;;;;N;;;;; +1B2E8;NUSHU CHARACTER-1B2E8;Lo;0;L;;;;;N;;;;; +1B2E9;NUSHU CHARACTER-1B2E9;Lo;0;L;;;;;N;;;;; +1B2EA;NUSHU CHARACTER-1B2EA;Lo;0;L;;;;;N;;;;; +1B2EB;NUSHU CHARACTER-1B2EB;Lo;0;L;;;;;N;;;;; +1B2EC;NUSHU CHARACTER-1B2EC;Lo;0;L;;;;;N;;;;; +1B2ED;NUSHU CHARACTER-1B2ED;Lo;0;L;;;;;N;;;;; +1B2EE;NUSHU CHARACTER-1B2EE;Lo;0;L;;;;;N;;;;; +1B2EF;NUSHU CHARACTER-1B2EF;Lo;0;L;;;;;N;;;;; +1B2F0;NUSHU CHARACTER-1B2F0;Lo;0;L;;;;;N;;;;; +1B2F1;NUSHU CHARACTER-1B2F1;Lo;0;L;;;;;N;;;;; +1B2F2;NUSHU CHARACTER-1B2F2;Lo;0;L;;;;;N;;;;; +1B2F3;NUSHU CHARACTER-1B2F3;Lo;0;L;;;;;N;;;;; +1B2F4;NUSHU CHARACTER-1B2F4;Lo;0;L;;;;;N;;;;; +1B2F5;NUSHU CHARACTER-1B2F5;Lo;0;L;;;;;N;;;;; +1B2F6;NUSHU CHARACTER-1B2F6;Lo;0;L;;;;;N;;;;; +1B2F7;NUSHU CHARACTER-1B2F7;Lo;0;L;;;;;N;;;;; +1B2F8;NUSHU CHARACTER-1B2F8;Lo;0;L;;;;;N;;;;; +1B2F9;NUSHU CHARACTER-1B2F9;Lo;0;L;;;;;N;;;;; +1B2FA;NUSHU CHARACTER-1B2FA;Lo;0;L;;;;;N;;;;; +1B2FB;NUSHU CHARACTER-1B2FB;Lo;0;L;;;;;N;;;;; 1BC00;DUPLOYAN LETTER H;Lo;0;L;;;;;N;;;;; 1BC01;DUPLOYAN LETTER X;Lo;0;L;;;;;N;;;;; 1BC02;DUPLOYAN LETTER P;Lo;0;L;;;;;N;;;;; @@ -28269,6 +29217,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F248;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557;So;0;L;<compat> 3014 6557 3015;;;;N;;;;; 1F250;CIRCLED IDEOGRAPH ADVANTAGE;So;0;L;<circle> 5F97;;;;N;;;;; 1F251;CIRCLED IDEOGRAPH ACCEPT;So;0;L;<circle> 53EF;;;;N;;;;; +1F260;ROUNDED SYMBOL FOR FU;So;0;ON;;;;;N;;;;; +1F261;ROUNDED SYMBOL FOR LU;So;0;ON;;;;;N;;;;; +1F262;ROUNDED SYMBOL FOR SHOU;So;0;ON;;;;;N;;;;; +1F263;ROUNDED SYMBOL FOR XI;So;0;ON;;;;;N;;;;; +1F264;ROUNDED SYMBOL FOR SHUANGXI;So;0;ON;;;;;N;;;;; +1F265;ROUNDED SYMBOL FOR CAI;So;0;ON;;;;;N;;;;; 1F300;CYCLONE;So;0;ON;;;;;N;;;;; 1F301;FOGGY;So;0;ON;;;;;N;;;;; 1F302;CLOSED UMBRELLA;So;0;ON;;;;;N;;;;; @@ -29248,6 +30202,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D0;PLACE OF WORSHIP;So;0;ON;;;;;N;;;;; 1F6D1;OCTAGONAL SIGN;So;0;ON;;;;;N;;;;; 1F6D2;SHOPPING TROLLEY;So;0;ON;;;;;N;;;;; +1F6D3;STUPA;So;0;ON;;;;;N;;;;; +1F6D4;PAGODA;So;0;ON;;;;;N;;;;; 1F6E0;HAMMER AND WRENCH;So;0;ON;;;;;N;;;;; 1F6E1;SHIELD;So;0;ON;;;;;N;;;;; 1F6E2;OIL DRUM;So;0;ON;;;;;N;;;;; @@ -29268,6 +30224,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6F4;SCOOTER;So;0;ON;;;;;N;;;;; 1F6F5;MOTOR SCOOTER;So;0;ON;;;;;N;;;;; 1F6F6;CANOE;So;0;ON;;;;;N;;;;; +1F6F7;SLED;So;0;ON;;;;;N;;;;; +1F6F8;FLYING SAUCER;So;0;ON;;;;;N;;;;; 1F700;ALCHEMICAL SYMBOL FOR QUINTESSENCE;So;0;ON;;;;;N;;;;; 1F701;ALCHEMICAL SYMBOL FOR AIR;So;0;ON;;;;;N;;;;; 1F702;ALCHEMICAL SYMBOL FOR FIRE;So;0;ON;;;;;N;;;;; @@ -29617,6 +30575,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F8AB;RIGHTWARDS FRONT-TILTED SHADOWED WHITE ARROW;So;0;ON;;;;;N;;;;; 1F8AC;WHITE ARROW SHAFT WIDTH ONE;So;0;ON;;;;;N;;;;; 1F8AD;WHITE ARROW SHAFT WIDTH TWO THIRDS;So;0;ON;;;;;N;;;;; +1F900;CIRCLED CROSS FORMEE WITH FOUR DOTS;So;0;ON;;;;;N;;;;; +1F901;CIRCLED CROSS FORMEE WITH TWO DOTS;So;0;ON;;;;;N;;;;; +1F902;CIRCLED CROSS FORMEE;So;0;ON;;;;;N;;;;; +1F903;LEFT HALF CIRCLE WITH FOUR DOTS;So;0;ON;;;;;N;;;;; +1F904;LEFT HALF CIRCLE WITH THREE DOTS;So;0;ON;;;;;N;;;;; +1F905;LEFT HALF CIRCLE WITH TWO DOTS;So;0;ON;;;;;N;;;;; +1F906;LEFT HALF CIRCLE WITH DOT;So;0;ON;;;;;N;;;;; +1F907;LEFT HALF CIRCLE;So;0;ON;;;;;N;;;;; +1F908;DOWNWARD FACING HOOK;So;0;ON;;;;;N;;;;; +1F909;DOWNWARD FACING NOTCHED HOOK;So;0;ON;;;;;N;;;;; +1F90A;DOWNWARD FACING HOOK WITH DOT;So;0;ON;;;;;N;;;;; +1F90B;DOWNWARD FACING NOTCHED HOOK WITH DOT;So;0;ON;;;;;N;;;;; 1F910;ZIPPER-MOUTH FACE;So;0;ON;;;;;N;;;;; 1F911;MONEY-MOUTH FACE;So;0;ON;;;;;N;;;;; 1F912;FACE WITH THERMOMETER;So;0;ON;;;;;N;;;;; @@ -29632,6 +30602,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F91C;RIGHT-FACING FIST;So;0;ON;;;;;N;;;;; 1F91D;HANDSHAKE;So;0;ON;;;;;N;;;;; 1F91E;HAND WITH INDEX AND MIDDLE FINGERS CROSSED;So;0;ON;;;;;N;;;;; +1F91F;I LOVE YOU HAND SIGN;So;0;ON;;;;;N;;;;; 1F920;FACE WITH COWBOY HAT;So;0;ON;;;;;N;;;;; 1F921;CLOWN FACE;So;0;ON;;;;;N;;;;; 1F922;NAUSEATED FACE;So;0;ON;;;;;N;;;;; @@ -29640,7 +30611,17 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F925;LYING FACE;So;0;ON;;;;;N;;;;; 1F926;FACE PALM;So;0;ON;;;;;N;;;;; 1F927;SNEEZING FACE;So;0;ON;;;;;N;;;;; +1F928;FACE WITH ONE EYEBROW RAISED;So;0;ON;;;;;N;;;;; +1F929;GRINNING FACE WITH STAR EYES;So;0;ON;;;;;N;;;;; +1F92A;GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE;So;0;ON;;;;;N;;;;; +1F92B;FACE WITH FINGER COVERING CLOSED LIPS;So;0;ON;;;;;N;;;;; +1F92C;SERIOUS FACE WITH SYMBOLS COVERING MOUTH;So;0;ON;;;;;N;;;;; +1F92D;SMILING FACE WITH SMILING EYES AND HAND COVERING MOUTH;So;0;ON;;;;;N;;;;; +1F92E;FACE WITH OPEN MOUTH VOMITING;So;0;ON;;;;;N;;;;; +1F92F;SHOCKED FACE WITH EXPLODING HEAD;So;0;ON;;;;;N;;;;; 1F930;PREGNANT WOMAN;So;0;ON;;;;;N;;;;; +1F931;BREAST-FEEDING;So;0;ON;;;;;N;;;;; +1F932;PALMS UP TOGETHER;So;0;ON;;;;;N;;;;; 1F933;SELFIE;So;0;ON;;;;;N;;;;; 1F934;PRINCE;So;0;ON;;;;;N;;;;; 1F935;MAN IN TUXEDO;So;0;ON;;;;;N;;;;; @@ -29665,6 +30646,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F949;THIRD PLACE MEDAL;So;0;ON;;;;;N;;;;; 1F94A;BOXING GLOVE;So;0;ON;;;;;N;;;;; 1F94B;MARTIAL ARTS UNIFORM;So;0;ON;;;;;N;;;;; +1F94C;CURLING STONE;So;0;ON;;;;;N;;;;; 1F950;CROISSANT;So;0;ON;;;;;N;;;;; 1F951;AVOCADO;So;0;ON;;;;;N;;;;; 1F952;CUCUMBER;So;0;ON;;;;;N;;;;; @@ -29680,6 +30662,19 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F95C;PEANUTS;So;0;ON;;;;;N;;;;; 1F95D;KIWIFRUIT;So;0;ON;;;;;N;;;;; 1F95E;PANCAKES;So;0;ON;;;;;N;;;;; +1F95F;DUMPLING;So;0;ON;;;;;N;;;;; +1F960;FORTUNE COOKIE;So;0;ON;;;;;N;;;;; +1F961;TAKEOUT BOX;So;0;ON;;;;;N;;;;; +1F962;CHOPSTICKS;So;0;ON;;;;;N;;;;; +1F963;BOWL WITH SPOON;So;0;ON;;;;;N;;;;; +1F964;CUP WITH STRAW;So;0;ON;;;;;N;;;;; +1F965;COCONUT;So;0;ON;;;;;N;;;;; +1F966;BROCCOLI;So;0;ON;;;;;N;;;;; +1F967;PIE;So;0;ON;;;;;N;;;;; +1F968;PRETZEL;So;0;ON;;;;;N;;;;; +1F969;CUT OF MEAT;So;0;ON;;;;;N;;;;; +1F96A;SANDWICH;So;0;ON;;;;;N;;;;; +1F96B;CANNED FOOD;So;0;ON;;;;;N;;;;; 1F980;CRAB;So;0;ON;;;;;N;;;;; 1F981;LION FACE;So;0;ON;;;;;N;;;;; 1F982;SCORPION;So;0;ON;;;;;N;;;;; @@ -29698,7 +30693,36 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F98F;RHINOCEROS;So;0;ON;;;;;N;;;;; 1F990;SHRIMP;So;0;ON;;;;;N;;;;; 1F991;SQUID;So;0;ON;;;;;N;;;;; +1F992;GIRAFFE FACE;So;0;ON;;;;;N;;;;; +1F993;ZEBRA FACE;So;0;ON;;;;;N;;;;; +1F994;HEDGEHOG;So;0;ON;;;;;N;;;;; +1F995;SAUROPOD;So;0;ON;;;;;N;;;;; +1F996;T-REX;So;0;ON;;;;;N;;;;; +1F997;CRICKET;So;0;ON;;;;;N;;;;; 1F9C0;CHEESE WEDGE;So;0;ON;;;;;N;;;;; +1F9D0;FACE WITH MONOCLE;So;0;ON;;;;;N;;;;; +1F9D1;ADULT;So;0;ON;;;;;N;;;;; +1F9D2;CHILD;So;0;ON;;;;;N;;;;; +1F9D3;OLDER ADULT;So;0;ON;;;;;N;;;;; +1F9D4;BEARDED PERSON;So;0;ON;;;;;N;;;;; +1F9D5;PERSON WITH HEADSCARF;So;0;ON;;;;;N;;;;; +1F9D6;PERSON IN STEAMY ROOM;So;0;ON;;;;;N;;;;; +1F9D7;PERSON CLIMBING;So;0;ON;;;;;N;;;;; +1F9D8;PERSON IN LOTUS POSITION;So;0;ON;;;;;N;;;;; +1F9D9;MAGE;So;0;ON;;;;;N;;;;; +1F9DA;FAIRY;So;0;ON;;;;;N;;;;; +1F9DB;VAMPIRE;So;0;ON;;;;;N;;;;; +1F9DC;MERPERSON;So;0;ON;;;;;N;;;;; +1F9DD;ELF;So;0;ON;;;;;N;;;;; +1F9DE;GENIE;So;0;ON;;;;;N;;;;; +1F9DF;ZOMBIE;So;0;ON;;;;;N;;;;; +1F9E0;BRAIN;So;0;ON;;;;;N;;;;; +1F9E1;ORANGE HEART;So;0;ON;;;;;N;;;;; +1F9E2;BILLED CAP;So;0;ON;;;;;N;;;;; +1F9E3;SCARF;So;0;ON;;;;;N;;;;; +1F9E4;GLOVES;So;0;ON;;;;;N;;;;; +1F9E5;COAT;So;0;ON;;;;;N;;;;; +1F9E6;SOCKS;So;0;ON;;;;;N;;;;; 20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;; 2A6D6;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;; 2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;; @@ -29707,6 +30731,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2B81D;<CJK Ideograph Extension D, Last>;Lo;0;L;;;;;N;;;;; 2B820;<CJK Ideograph Extension E, First>;Lo;0;L;;;;;N;;;;; 2CEA1;<CJK Ideograph Extension E, Last>;Lo;0;L;;;;;N;;;;; +2CEB0;<CJK Ideograph Extension F, First>;Lo;0;L;;;;;N;;;;; +2EBE0;<CJK Ideograph Extension F, Last>;Lo;0;L;;;;;N;;;;; 2F800;CJK COMPATIBILITY IDEOGRAPH-2F800;Lo;0;L;4E3D;;;;N;;;;; 2F801;CJK COMPATIBILITY IDEOGRAPH-2F801;Lo;0;L;4E38;;;;N;;;;; 2F802;CJK COMPATIBILITY IDEOGRAPH-2F802;Lo;0;L;4E41;;;;N;;;;; diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index fefd7d3b70..5b8763f576 100755 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -186,7 +186,7 @@ gen_static(Fd) -> " {U,L} -> #{upper=>U,lower=>L,title=>U,fold=>L};\n" " {U,L,T,F} -> #{upper=>U,lower=>L,title=>T,fold=>F}\n" " end.\n\n"), - io:put_chars(Fd, "spec_version() -> {9,0}.\n\n\n"), + io:put_chars(Fd, "spec_version() -> {10,0}.\n\n\n"), io:put_chars(Fd, "class(Codepoint) -> {CCC,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), io:put_chars(Fd, "-spec uppercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index f062c7fe6e..8a83cdec1e 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.4 +STDLIB_VSN = 3.4.1 diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl index 1ca60ea73b..888cb71f51 100644 --- a/lib/syntax_tools/src/erl_tidy.erl +++ b/lib/syntax_tools/src/erl_tidy.erl @@ -301,6 +301,8 @@ file(Name, Opts) -> {Child, ok} -> ok; {Child, {error, Reason}} -> + exit(Reason); + {'EXIT', Child, Reason} -> exit(Reason) end. diff --git a/lib/syntax_tools/test/syntax_tools_SUITE.erl b/lib/syntax_tools/test/syntax_tools_SUITE.erl index 868f43b8ee..ae2c67c03e 100644 --- a/lib/syntax_tools/test/syntax_tools_SUITE.erl +++ b/lib/syntax_tools/test/syntax_tools_SUITE.erl @@ -239,6 +239,12 @@ t_erl_tidy(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), File = filename:join(DataDir,"erl_tidy_tilde.erl"), ok = erl_tidy:file(File, [{stdout, true}]), + + %% OTP-14471. + Old = process_flag(trap_exit, true), + NonExisting = filename:join(DataDir,"non_existing_file.erl"), + {'EXIT',{error,{0,file,enoent}}} = (catch erl_tidy:file(NonExisting)), + true = process_flag(trap_exit, Old), ok. test_comment_scan([],_) -> ok; diff --git a/lib/tools/doc/src/lcnt.xml b/lib/tools/doc/src/lcnt.xml index 31e5c241e9..5bdfc60448 100644 --- a/lib/tools/doc/src/lcnt.xml +++ b/lib/tools/doc/src/lcnt.xml @@ -109,14 +109,6 @@ statistics. If the server held any lock statistics data before the collect then that data is lost. </p> - <note> - <p> - When collection occurs the runtime system transitions to a single thread, - blocking all other threads. No other tasks will be scheduled during this - operation. Depending on the size of the data this might take a long time - (several seconds) and cause timeouts in the system. - </p> - </note> </desc> </func> @@ -322,24 +314,22 @@ <func> <name>apply(Fun) -> term()</name> <fsummary>Same as <c>apply(Fun, [])</c>.</fsummary> + <type> + <v>Fun = fun()</v> + </type> <desc> <p>Same as <c>apply(Fun, [])</c>.</p> </desc> </func> <func> <name>apply(Fun, Args) -> term()</name> - <fsummary>Clears counters, applies function and collects the profiling results.</fsummary> + <fsummary>Same as <c>apply(Module, Function, Args)</c>.</fsummary> <type> <v>Fun = fun()</v> <v>Args = [term()]</v> </type> <desc> - <p> Clears the lock counters and then setups the instrumentation to save all destroyed locks. - After setup the fun is called, passing the elements in <c>Args</c> as arguments. - When the fun returns the statistics are immediately collected to the server. After the - collection the instrumentation is returned to its previous behavior. - The result of the applied fun is returned. - </p> + <p>Same as <c>apply(Module, Function, Args)</c>.</p> </desc> </func> <func> @@ -357,6 +347,13 @@ collection the instrumentation is returned to its previous behavior. The result of the applied function is returned. </p> + <warning> + <p> + This function should only be used for micro-benchmarks; it sets <c>copy_save</c> + to <c>true</c> for the duration of the call, which can quickly lead to running + out of memory. + </p> + </warning> </desc> </func> @@ -429,6 +426,68 @@ <desc> <p>Clear the internal counters. Same as <c>lcnt:clear(Node)</c>.</p></desc> </func> + <func> + <name>rt_mask() -> [category_atom()]</name> + <fsummary>Same as <c>rt_mask(node())</c>.</fsummary> + <desc><p>Same as <c>rt_mask(node())</c>.</p></desc> + </func> + + <func> + <name>rt_mask(Node) -> [category_atom()]</name> + <fsummary>Returns the current lock category mask.</fsummary> + <type> + <v>Node = node()</v> + </type> + <desc> + <p> + Refer to <c>rt_mask/2</c> for a list of valid categories. All + categories are enabled by default. + </p> + </desc> + </func> + + <func> + <name>rt_mask(Categories) -> ok | {error, copy_save_enabled}</name> + <fsummary>Same as <c>rt_mask(node(), Categories)</c>.</fsummary> + <type> + <v>Categories = [atom()]</v> + </type> + <desc><p>Same as <c>rt_mask(node(), Categories)</c>.</p></desc> + </func> + + <func> + <name>rt_mask(Node, Categories) -> ok | {error, copy_save_enabled}</name> + <fsummary>Changes the lock category mask.</fsummary> + <type> + <v>Node = node()</v> + <v>Categories = [atom()]</v> + </type> + <desc> + <p> + Sets the lock category mask to the given categories. + </p> + <p> + This will fail if the <c>copy_save</c> option is enabled; see + <c>lcnt:rt_opt/2</c>. + </p> + <p>Valid categories are:</p> + <list> + <item><c>allocator</c></item> + <item><c>db</c> (ETS tables)</item> + <item><c>debug</c></item> + <item><c>distribution</c></item> + <item><c>generic</c></item> + <item><c>io</c></item> + <item><c>process</c></item> + <item><c>scheduler</c></item> + </list> + <p> + This list is subject to change at any time, as is the category any given lock + may belong to. + </p> + </desc> + </func> + <func> <name>rt_opt({Type, bool()}) -> bool()</name> <fsummary>Same as <c>rt_opt(node(), {Type, Opt})</c>.</fsummary> @@ -442,16 +501,25 @@ <v>Type = copy_save | process_locks</v> </type> <desc> - <p>Changes the lock counter behavior and returns the previous behaviour.</p> <p>Option description:</p> <taglist> <tag><c>{copy_save, bool()}</c></tag> - <item>Enable statistics saving from destroyed locks by copying. This might consume a lot of memory. + <item>Retains the statistics of destroyed locks. <br/>Default: <c>false</c> + <warning> + <p> + This option will use a lot of memory when enabled, which must be + reclaimed with <c>lcnt:rt_clear</c>. Note that it makes no distinction + between locks that were destroyed and locks for which counting was + disabled, so enabling this option will disable changes to the lock + category mask. + </p> + </warning> </item> <tag><c>{process_locks, bool()}</c></tag> - <item>Profile process locks. + <item>Profile process locks, equal to adding <c>process</c> to the lock category mask; + see <c>lcnt:rt_mask/2</c> <br/>Default: <c>true</c> </item> </taglist> diff --git a/lib/tools/doc/src/lcnt_chapter.xml b/lib/tools/doc/src/lcnt_chapter.xml index c73fcb31e0..24b58136aa 100644 --- a/lib/tools/doc/src/lcnt_chapter.xml +++ b/lib/tools/doc/src/lcnt_chapter.xml @@ -29,7 +29,7 @@ <approved>nobody</approved> <checked>no</checked> <date>2009-11-26</date> - <rev>PA1</rev> + <rev>PA2</rev> <file>lcnt_chapter.xml</file> </header> <p> @@ -97,8 +97,11 @@ ok ok </pre> <p> - Another way to to profile a specific function is to use <c>lcnt:apply/3</c> or <c>lcnt:apply/1</c> which does <c>lcnt:clear/0</c> before the function and <c>lcnt:collect/0</c> after its invocation. - It also sets <c>copy_save</c> to <c>true</c> for the duration of the function call + Another way to to profile a specific function is to use <c>lcnt:apply/3</c> or <c>lcnt:apply/1</c> + which does <c>lcnt:clear/0</c> before the function and <c>lcnt:collect/0</c> after its invocation. + This method should only be used in micro-benchmarks since it sets <c>copy_save</c> to <c>true</c> + for the duration of the function call, which may cause the emulator to run out of memory if + attempted under load. </p> <pre> Erlang R13B03 (erts-5.7.4) [source] [smp:8:8] [rq:8] [async-threads:0] [hipe] diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index bdd5455354..f0df43bf2b 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,26 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 2.10.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + In OTP-20.0, the behavior of c, make, and ct_make was + changed so that in some cases the beam files by default + would be written to the directory where the source files + were found. This is now changed back to the old behavior + so beam files are by default written to current + directory.</p> + <p> + Own Id: OTP-14489 Aux Id: ERL-438 </p> + </item> + </list> + </section> + +</section> + <section><title>Tools 2.10</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 438abc2d29..e6e2050f29 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -900,6 +900,11 @@ resulting regexp is surrounded by \\_< and \\_>." "display" "display_nl" "display_string" + "dist_get_stat" + "dist_ctrl_get_data" + "dist_ctrl_get_data_notification" + "dist_ctrl_input_handler" + "dist_ctrl_put_data" "dist_exit" "dlink" "dmonitor_node" @@ -3642,8 +3647,10 @@ The return value is a string of the form \"foo/1\"." (error nil))))) -;; Keeping erlang-get-function-under-point for backward compatibility. -;; It is used by erldoc.el and maybe other code out there. +;; erlang-get-function-under-point is replaced by +;; erlang-get-identifier-at-point as far as internal erlang.el usage +;; is concerned. But it is kept for backward compatibility. It is +;; used by erldoc.el and maybe other code out there. (defun erlang-get-function-under-point () "Return the module and function under the point, or nil. @@ -4881,7 +4888,12 @@ considered first when it is time to jump to the definition.") '(progn (cl-defmethod xref-backend-identifier-at-point ((_backend (eql erlang-etags))) - (erlang-id-to-string (erlang-get-identifier-at-point))) + (if (eq this-command 'xref-find-references) + (if (use-region-p) + (buffer-substring-no-properties (region-beginning) + (region-end)) + (thing-at-point 'symbol)) + (erlang-id-to-string (erlang-get-identifier-at-point)))) (cl-defmethod xref-backend-definitions ((_backend (eql erlang-etags)) identifier) diff --git a/lib/tools/src/fprof.erl b/lib/tools/src/fprof.erl index d1a4624419..436f68d12b 100644 --- a/lib/tools/src/fprof.erl +++ b/lib/tools/src/fprof.erl @@ -2636,22 +2636,32 @@ funcstat_pd(Pid, Func1, Func0, Clocks) -> #funcstat{callers_sum = CallersSum, callers = Callers} = FuncstatCallers -> FuncstatCallers#funcstat{ - callers_sum = clocks_sum(CallersSum, Clocks, Func0), - callers = [Clocks#clocks{id = Func1} | Callers]} - end), + callers_sum = clocks_sum(CallersSum, Clocks, Func0), + callers = insert_call(Clocks, Func1, Callers)} + end), put({Pid, Func1}, case get({Pid, Func1}) of undefined -> - #funcstat{callers_sum = #clocks{id = Func1}, + #funcstat{callers_sum = #clocks{id = Func1}, called_sum = Clocks#clocks{id = Func1}, called = [Clocks#clocks{id = Func0}]}; #funcstat{called_sum = CalledSum, called = Called} = FuncstatCalled -> FuncstatCalled#funcstat{ called_sum = clocks_sum(CalledSum, Clocks, Func1), - called = [Clocks#clocks{id = Func0} | Called]} + called = insert_call(Clocks, Func0, Called)} end). +insert_call(Clocks, Func, ClocksList) -> + insert_call(Clocks, Func, ClocksList, []). + +insert_call(Clocks, Func, [#clocks{id = Func} = C | T], Acc) -> + [clocks_sum(C, Clocks, Func) | T ++ Acc]; +insert_call(Clocks, Func, [H | T], Acc) -> + insert_call(Clocks, Func, T, [H | Acc]); +insert_call(Clocks, Func, [], Acc) -> + [Clocks#clocks{id = Func} | Acc]. + %% Sort a list of funcstat records, diff --git a/lib/tools/src/lcnt.erl b/lib/tools/src/lcnt.erl index d881fedbd5..139b3d8a4a 100644 --- a/lib/tools/src/lcnt.erl +++ b/lib/tools/src/lcnt.erl @@ -34,8 +34,11 @@ -export([start/0, stop/0]). -%% erts_debug:lock_counters api --export([rt_collect/0, +%% erts_debug:lcnt_xxx api +-export([rt_mask/0, + rt_mask/1, + rt_mask/2, + rt_collect/0, rt_collect/1, rt_clear/0, rt_clear/1, @@ -134,27 +137,61 @@ start_internal() -> %% -------------------------------------------------------------------- %% %% -%% API erts_debug:lock_counters +%% API erts_debug:lcnt_xxx %% %% -------------------------------------------------------------------- %% -rt_collect() -> - erts_debug:lock_counters(info). +rt_mask(Node, Categories) when is_atom(Node), is_list(Categories) -> + rpc:call(Node, lcnt, rt_mask, [Categories]). + +rt_mask(Node) when is_atom(Node) -> + rpc:call(Node, lcnt, rt_mask, []); + +rt_mask(Categories) when is_list(Categories) -> + case erts_debug:lcnt_control(copy_save) of + false -> + erts_debug:lcnt_control(mask, Categories); + true -> + {error, copy_save_enabled} + end. + +rt_mask() -> + erts_debug:lcnt_control(mask). rt_collect(Node) -> - rpc:call(Node, erts_debug, lock_counters, [info]). + rpc:call(Node, lcnt, rt_collect, []). +rt_collect() -> + erts_debug:lcnt_collect(). +rt_clear(Node) -> + rpc:call(Node, lcnt, rt_clear, []). rt_clear() -> - erts_debug:lock_counters(clear). + erts_debug:lcnt_clear(). -rt_clear(Node) -> - rpc:call(Node, erts_debug, lock_counters, [clear]). +rt_opt(Node, Arg) -> + rpc:call(Node, lcnt, rt_opt, [Arg]). -rt_opt({Type, Opt}) -> - erts_debug:lock_counters({Type, Opt}). +%% Compatibility shims for the "process/port_locks" options mentioned in the +%% manual. +rt_opt({process_locks, Enable}) -> + toggle_category(process, Enable); +rt_opt({port_locks, Enable}) -> + toggle_category(io, Enable); -rt_opt(Node, {Type, Opt}) -> - rpc:call(Node, erts_debug, lock_counters, [{Type, Opt}]). +rt_opt({Type, NewVal}) -> + PreviousVal = erts_debug:lcnt_control(Type), + erts_debug:lcnt_control(Type, NewVal), + PreviousVal. + +toggle_category(Category, true) -> + PreviousMask = erts_debug:lcnt_control(mask), + erts_debug:lcnt_control(mask, [Category | PreviousMask]), + lists:member(Category, PreviousMask); + +toggle_category(Category, false) -> + PreviousMask = erts_debug:lcnt_control(mask), + erts_debug:lcnt_control(mask, lists:delete(Category, PreviousMask)), + lists:member(Category, PreviousMask). %% -------------------------------------------------------------------- %% %% @@ -192,13 +229,9 @@ call(Msg) -> gen_server:call(?MODULE, Msg, infinity). %% -------------------------------------------------------------------- %% apply(M,F,As) when is_atom(M), is_atom(F), is_list(As) -> - ok = start_internal(), - Opt = lcnt:rt_opt({copy_save, true}), - lcnt:clear(), - Res = erlang:apply(M,F,As), - lcnt:collect(), - lcnt:rt_opt({copy_save, Opt}), - Res. + apply(fun() -> + erlang:apply(M,F,As) + end). apply(Fun) when is_function(Fun) -> lcnt:apply(Fun, []). @@ -209,7 +242,9 @@ apply(Fun, As) when is_function(Fun) -> lcnt:clear(), Res = erlang:apply(Fun, As), lcnt:collect(), - lcnt:rt_opt({copy_save, Opt}), + %% _ is bound to silence a dialyzer warning; it used to fail silently and + %% we don't want to change the error semantics. + _ = lcnt:rt_opt({copy_save, Opt}), Res. all_conflicts() -> all_conflicts(time). diff --git a/lib/tools/src/make.erl b/lib/tools/src/make.erl index ce30156db6..6554d338af 100644 --- a/lib/tools/src/make.erl +++ b/lib/tools/src/make.erl @@ -267,15 +267,47 @@ include_opt([]) -> recompile(File, true, _Load, _Opts) -> io:format("Out of date: ~ts\n",[File]); -recompile(File, false, noload, Opts) -> +recompile(File, false, Load, Opts) -> io:format("Recompile: ~ts\n",[File]), - compile:file(File, [report_errors, report_warnings, error_summary |Opts]); -recompile(File, false, load, Opts) -> - io:format("Recompile: ~ts\n",[File]), - c:c(File, Opts); -recompile(File, false, netload, Opts) -> - io:format("Recompile: ~ts\n",[File]), - c:nc(File, Opts). + case compile:file(File, [report_errors, report_warnings |Opts]) of + Ok when is_tuple(Ok), element(1,Ok)==ok -> + maybe_load(element(2,Ok), Load, Opts); + _Error -> + error + end. + +maybe_load(_Mod, noload, _Opts) -> + ok; +maybe_load(Mod, Load, Opts) -> + %% We have compiled File with options Opts. Find out where the + %% output file went to, and load it. + case compile:output_generated(Opts) of + true -> + Dir = proplists:get_value(outdir,Opts,"."), + do_load(Dir, Mod, Load); + false -> + io:format("** Warning: No object file created - nothing loaded **~n"), + ok + end. + +do_load(Dir, Mod, load) -> + code:purge(Mod), + case code:load_abs(filename:join(Dir, Mod),Mod) of + {module,Mod} -> + {ok,Mod}; + Other -> + Other + end; +do_load(Dir, Mod, netload) -> + Obj = atom_to_list(Mod) ++ code:objfile_extension(), + Fname = filename:join(Dir, Obj), + case file:read_file(Fname) of + {ok,Bin} -> + rpc:eval_everywhere(code,load_binary,[Mod,Fname,Bin]), + {ok,Mod}; + Other -> + Other + end. exists(File) -> case file:read_file_info(File) of diff --git a/lib/tools/src/tools.app.src b/lib/tools/src/tools.app.src index 12f0cfd2df..8beef49bf9 100644 --- a/lib/tools/src/tools.app.src +++ b/lib/tools/src/tools.app.src @@ -41,6 +41,6 @@ ] }, {runtime_dependencies, ["stdlib-3.1","runtime_tools-1.8.14", - "kernel-3.0","erts-7.0","compiler-5.0"]} + "kernel-5.4","erts-9.1","compiler-5.0"]} ] }. diff --git a/lib/tools/test/lcnt_SUITE.erl b/lib/tools/test/lcnt_SUITE.erl index af3ce88fdd..146c915087 100644 --- a/lib/tools/test/lcnt_SUITE.erl +++ b/lib/tools/test/lcnt_SUITE.erl @@ -151,10 +151,9 @@ t_swap_keys_file([File|Files]) -> %% Simple smoke test of actual lock-counting, if running on %% a run-time with lock-counting enabled. - smoke_lcnt(Config) -> - case erlang:system_info(build_type) of - lcnt -> + case catch erlang:system_info(lock_counting) of + true -> do_smoke_lcnt(Config); _ -> {skip,"Lock counting is not enabled"} diff --git a/lib/tools/test/make_SUITE.erl b/lib/tools/test/make_SUITE.erl index 2f6fe1c732..02da4f4ace 100644 --- a/lib/tools/test/make_SUITE.erl +++ b/lib/tools/test/make_SUITE.erl @@ -36,7 +36,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [make_all, make_files, recompile_on_changed_include, + [make_all, make_files, load, netload, recompile_on_changed_include, emake_opts, {group, otp_6057}]. groups() -> @@ -55,6 +55,21 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> otp_6057_end(Config). +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(netload,_Config) -> + %% Stop slave - in case of failure + Nodes = nodes(), + case [N || N <- Nodes, + "make_SUITE_netload" == hd(string:lexemes(atom_to_list(N),"@"))] of + [Node] -> + ct_slave:stop(Node); + _ -> + ok + end; +end_per_testcase(_,_Config) -> + ok. test_files() -> ["test1", "test2", "test3", "test4"]. @@ -83,6 +98,32 @@ make_files(Config) when is_list(Config) -> ensure_no_messages(), ok. +load(Config) -> + Current = prepare_data_dir(Config), + code:purge(test1), + code:delete(test1), + false = code:is_loaded(test1), + up_to_date = make:files([test1], [load]), + {file,_} = code:is_loaded(test1), + file:set_cwd(Current), + ensure_no_messages(), + ok. + +netload(Config) -> + Current = prepare_data_dir(Config), + code:purge(test1), + code:delete(test1), + false = code:is_loaded(test1), + {ok,Node} = ct_slave:start(make_SUITE_netload), + up_to_date = make:files([test1], [netload]), + timer:sleep(1000), % async, so give some time + {file,F} = code:is_loaded(test1), + {file,F} = rpc:call(Node,code,is_loaded,[test1]), + ct_slave:stop(Node), + file:set_cwd(Current), + ensure_no_messages(), + ok. + recompile_on_changed_include(Config) -> Current = prepare_data_dir(Config), diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index 8aa7814e1d..831d850217 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 2.10 +TOOLS_VSN = 2.10.1 diff --git a/lib/wx/api_gen/README b/lib/wx/api_gen/README index dd0c49d227..200ef4c856 100644 --- a/lib/wx/api_gen/README +++ b/lib/wx/api_gen/README @@ -3,12 +3,13 @@ API GENERATION: Users of wxErlang should not normally need to regenerate the generated code, as it is checked in by wxErlang developers, when changes are made. - Code checked in is currently generated from wxwidgets 2.8.10. + Code checked in is currently generated from wxwidgets 2.8.12. REQUIREMENTS: The code generation requires doxygen (1.4.6) which is used to parse wxWidgets c++ headers and generate xml files (in wx_xml/). + 2017-08-16 doxygen 1.8.11 is working with WXGTK_DIR=/ldisk/src/wxWidgets-2.8.12/include 2012-02-09 doxygen 1.7.4 is working fine diff --git a/lib/wx/api_gen/wx_doxygen.conf b/lib/wx/api_gen/wx_doxygen.conf index a96db00254..d6a0e9e6a1 100644 --- a/lib/wx/api_gen/wx_doxygen.conf +++ b/lib/wx/api_gen/wx_doxygen.conf @@ -71,12 +71,12 @@ WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- -INPUT = @WXGTK_DIR@/wx/ wx_extra/ +INPUT = @WXGTK_DIR@/wx/ @WXGTK_DIR@/../contrib/include/wx/stc/ wx_extra/ # FILE_PATTERNS = *.h RECURSIVE = YES EXCLUDE = EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = mac/* mgl/* msw/* os2/* x11/* gtk1/* cocoa/* motif/* msdos/* palmos/* private/* vms_x_fix.h +EXCLUDE_PATTERNS = */mac/* */dfb/* */mgl/* */msw/* */os2/* */x11/* */gtk1/* */cocoa/* */motif/* */msdos/* */palmos/* */private/* */univ/* */vms_x_fix.h EXAMPLE_PATH = EXAMPLE_PATTERNS = EXAMPLE_RECURSIVE = NO @@ -155,8 +155,6 @@ MAN_LINKS = NO #--------------------------------------------------------------------------- GENERATE_XML = YES XML_OUTPUT = ./wx_xml/ -XML_SCHEMA = -XML_DTD = XML_PROGRAMLISTING = NO #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output diff --git a/lib/wx/api_gen/wx_gen.erl b/lib/wx/api_gen/wx_gen.erl index 6979a600f3..aadfe4b111 100644 --- a/lib/wx/api_gen/wx_gen.erl +++ b/lib/wx/api_gen/wx_gen.erl @@ -501,10 +501,11 @@ parse_member2(_, _,M0) -> M0. add_param(InParam, Opts, M0) -> - Param0 = case InParam#param.name of - undefined -> InParam#param{name="val"}; + Param0 = case {InParam#param.name, InParam#param.type} of + {undefined, void} -> InParam#param{where=nowhere}; + {undefined,_} -> InParam#param{name="val"}; _ -> InParam - end, + end, Param = case Param0#param.type of #type{base={comp,_,_Comp}} -> Param0; #type{base={class,_Class}} -> Param0; diff --git a/lib/wx/api_gen/wxapi.conf b/lib/wx/api_gen/wxapi.conf index a0dfa61dd1..146c9fecc7 100644 --- a/lib/wx/api_gen/wxapi.conf +++ b/lib/wx/api_gen/wxapi.conf @@ -401,8 +401,8 @@ ['~wxGraphicsContext', 'Create', %%CreateFromNative CreateFromNativeWindow 'CreatePen','CreateBrush', - {'CreateRadialGradientBrush', [{deprecated, "!wxCHECK_VERSION(2,9,0)"}]}, - {'CreateLinearGradientBrush', [{deprecated, "!wxCHECK_VERSION(2,9,0)"}]}, + 'CreateRadialGradientBrush', + 'CreateLinearGradientBrush', 'CreateFont','CreateMatrix', 'CreatePath','Clip','ResetClip', 'DrawBitmap','DrawEllipse','DrawIcon', diff --git a/lib/wx/c_src/gen/wxe_funcs.cpp b/lib/wx/c_src/gen/wxe_funcs.cpp index 5425e9f3cb..a47d602337 100644 --- a/lib/wx/c_src/gen/wxe_funcs.cpp +++ b/lib/wx/c_src/gen/wxe_funcs.cpp @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -6177,7 +6177,6 @@ case wxGraphicsContext_CreateBrush: { // wxGraphicsContext::CreateBrush rt.addRef(getRef((void *)Result,memenv), "wxGraphicsBrush"); break; } -#if !wxCHECK_VERSION(2,9,0) case wxGraphicsContext_CreateRadialGradientBrush: { // wxGraphicsContext::CreateRadialGradientBrush wxGraphicsContext *This = (wxGraphicsContext *) getPtr(bp,memenv); bp += 4; bp += 4; /* Align */ @@ -6201,8 +6200,6 @@ case wxGraphicsContext_CreateRadialGradientBrush: { // wxGraphicsContext::Create rt.addRef(getRef((void *)Result,memenv), "wxGraphicsBrush"); break; } -#endif -#if !wxCHECK_VERSION(2,9,0) case wxGraphicsContext_CreateLinearGradientBrush: { // wxGraphicsContext::CreateLinearGradientBrush wxGraphicsContext *This = (wxGraphicsContext *) getPtr(bp,memenv); bp += 4; bp += 4; /* Align */ @@ -6225,7 +6222,6 @@ case wxGraphicsContext_CreateLinearGradientBrush: { // wxGraphicsContext::Create rt.addRef(getRef((void *)Result,memenv), "wxGraphicsBrush"); break; } -#endif case wxGraphicsContext_CreateFont: { // wxGraphicsContext::CreateFont wxColour col= *wxBLACK; wxGraphicsContext *This = (wxGraphicsContext *) getPtr(bp,memenv); bp += 4; diff --git a/lib/wx/c_src/gen/wxe_macros.h b/lib/wx/c_src/gen/wxe_macros.h index f44fa57053..4c8e52def2 100644 --- a/lib/wx/c_src/gen/wxe_macros.h +++ b/lib/wx/c_src/gen/wxe_macros.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1540,10 +1540,10 @@ #define wxStaticBox_destroy 1637 #define wxStaticLine_new_2 1639 #define wxStaticLine_new_0 1640 -#define wxStaticLine_Create 1641 -#define wxStaticLine_IsVertical 1642 -#define wxStaticLine_GetDefaultSize 1643 -#define wxStaticLine_destroy 1644 +#define wxStaticLine_destruct 1641 +#define wxStaticLine_Create 1642 +#define wxStaticLine_IsVertical 1643 +#define wxStaticLine_GetDefaultSize 1644 #define wxListBox_new_3 1647 #define wxListBox_new_0 1648 #define wxListBox_destruct 1650 diff --git a/lib/wx/examples/demo/demo.erl b/lib/wx/examples/demo/demo.erl index 8fb70ad7b0..8f3291305b 100644 --- a/lib/wx/examples/demo/demo.erl +++ b/lib/wx/examples/demo/demo.erl @@ -411,7 +411,7 @@ find(Ed) -> keyWords() -> L = ["after","begin","case","try","cond","catch","andalso","orelse", - "end","fun","if","let","of","query","receive","when","bnot","not", + "end","fun","if","let","of","receive","when","bnot","not", "div","rem","band","and","bor","bxor","bsl","bsr","or","xor"], lists:flatten([K ++ " " || K <- L] ++ [0]). diff --git a/lib/wx/src/gen/wxGraphicsContext.erl b/lib/wx/src/gen/wxGraphicsContext.erl index 2d0271ac48..5d371ecd7a 100644 --- a/lib/wx/src/gen/wxGraphicsContext.erl +++ b/lib/wx/src/gen/wxGraphicsContext.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,8 +41,6 @@ -export([getRenderer/1,isNull/1,parent_class/1]). -export_type([wxGraphicsContext/0]). --deprecated([createLinearGradientBrush/7,createRadialGradientBrush/8]). - %% @hidden parent_class(wxGraphicsObject) -> true; parent_class(_Class) -> erlang:error({badtype, ?MODULE}). diff --git a/lib/wx/src/gen/wxe_debug.hrl b/lib/wx/src/gen/wxe_debug.hrl index 58cb5298e6..533f9f2df0 100644 --- a/lib/wx/src/gen/wxe_debug.hrl +++ b/lib/wx/src/gen/wxe_debug.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1491,10 +1491,10 @@ wxdebug_table() -> {1637, {wxStaticBox, 'Destroy', undefined}}, {1639, {wxStaticLine, new_2, 2}}, {1640, {wxStaticLine, new_0, 0}}, - {1641, {wxStaticLine, create, 2}}, - {1642, {wxStaticLine, isVertical, 0}}, - {1643, {wxStaticLine, getDefaultSize, 0}}, - {1644, {wxStaticLine, 'Destroy', undefined}}, + {1641, {wxStaticLine, destruct, 0}}, + {1642, {wxStaticLine, create, 2}}, + {1643, {wxStaticLine, isVertical, 0}}, + {1644, {wxStaticLine, getDefaultSize, 0}}, {1647, {wxListBox, new_3, 3}}, {1648, {wxListBox, new_0, 0}}, {1650, {wxListBox, destruct, 0}}, diff --git a/lib/wx/src/gen/wxe_funcs.hrl b/lib/wx/src/gen/wxe_funcs.hrl index af0cee0dcd..14b5545676 100644 --- a/lib/wx/src/gen/wxe_funcs.hrl +++ b/lib/wx/src/gen/wxe_funcs.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1488,10 +1488,10 @@ -define(wxStaticBox_destroy, 1637). -define(wxStaticLine_new_2, 1639). -define(wxStaticLine_new_0, 1640). --define(wxStaticLine_Create, 1641). --define(wxStaticLine_IsVertical, 1642). --define(wxStaticLine_GetDefaultSize, 1643). --define(wxStaticLine_destroy, 1644). +-define(wxStaticLine_destruct, 1641). +-define(wxStaticLine_Create, 1642). +-define(wxStaticLine_IsVertical, 1643). +-define(wxStaticLine_GetDefaultSize, 1644). -define(wxListBox_new_3, 1647). -define(wxListBox_new_0, 1648). -define(wxListBox_destruct, 1650). |